The Unified and Autonomous Development Platform
Fix issue #822 - Filter pojects/repositories by group/organization when importing from GitHub/GitLab/Gitea
Robin Shena81e1fb22413
diff --git a/server-core/src/main/java/io/onedev/server/imports/Importer.java b/server-core/src/main/java/io/onedev/server/imports/Importer.javadeleted file mode 100644index 753a49d4a9..0000000000--- a/server-core/src/main/java/io/onedev/server/imports/Importer.java+++ /dev/null@@ -1,45 +0,0 @@-package io.onedev.server.imports;--import java.io.Serializable;-import java.util.List;--import io.onedev.commons.utils.TaskLogger;-import io.onedev.server.util.ReflectionUtils;--public abstract class Importer<Where extends Serializable, What extends Serializable, How extends Serializable> implements Serializable {-- private static final long serialVersionUID = 1L;-- private final Class<Where> whereClass;-- private final Class<What> whatClass;-- private final Class<How> howClass;-- @SuppressWarnings("unchecked")- public Importer() {- List<Class<?>> typeArguments = ReflectionUtils.getTypeArguments(Importer.class, getClass());- whereClass = (Class<Where>) typeArguments.get(0);- whatClass = (Class<What>) typeArguments.get(1);- howClass = (Class<How>) typeArguments.get(2);- }-- public Class<Where> getWhereClass() {- return whereClass;- }-- public Class<What> getWhatClass() {- return whatClass;- }-- public Class<How> getHowClass() {- return howClass;- }-- public abstract String getName();-- public abstract What getWhat(Where where, TaskLogger logger);-- public abstract How getHow(Where where, What what, TaskLogger logger);--}diff --git a/server-core/src/main/java/io/onedev/server/imports/IssueImporter.java b/server-core/src/main/java/io/onedev/server/imports/IssueImporter.javaindex aa48d238f1..4b51085646 100644--- a/server-core/src/main/java/io/onedev/server/imports/IssueImporter.java+++ b/server-core/src/main/java/io/onedev/server/imports/IssueImporter.java@@ -1,19 +1,21 @@package io.onedev.server.imports;import java.io.Serializable;+import java.util.List;import javax.annotation.Nullable;import io.onedev.commons.utils.TaskLogger;import io.onedev.server.model.Project;+import io.onedev.server.web.util.ImportStep;-public abstract class IssueImporter<Where extends Serializable, What extends Serializable, How extends Serializable>- extends Importer<Where, What, How> {+public interface IssueImporter extends Serializable {- private static final long serialVersionUID = 1L;+ String getName();++ List<ImportStep<? extends Serializable>> getSteps();@Nullable- public abstract String doImport(Where where, What what, How how, Project project,- boolean retainIssueNumbers, boolean dryRun, TaskLogger logger);+ public abstract String doImport(Project project, boolean retainIssueNumbers, boolean dryRun, TaskLogger logger);}diff --git a/server-core/src/main/java/io/onedev/server/imports/IssueImporterContribution.java b/server-core/src/main/java/io/onedev/server/imports/IssueImporterContribution.javaindex adc178abcd..98128f8c43 100644--- a/server-core/src/main/java/io/onedev/server/imports/IssueImporterContribution.java+++ b/server-core/src/main/java/io/onedev/server/imports/IssueImporterContribution.java@@ -1,6 +1,5 @@package io.onedev.server.imports;-import java.io.Serializable;import java.util.Collection;import io.onedev.commons.loader.ExtensionPoint;@@ -8,7 +7,7 @@ import io.onedev.commons.loader.ExtensionPoint;@ExtensionPointpublic interface IssueImporterContribution {- Collection<IssueImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> getImporters();+ Collection<IssueImporter> getImporters();int getOrder();diff --git a/server-core/src/main/java/io/onedev/server/imports/ProjectImporter.java b/server-core/src/main/java/io/onedev/server/imports/ProjectImporter.javaindex c1b44d7a38..1972d0288e 100644--- a/server-core/src/main/java/io/onedev/server/imports/ProjectImporter.java+++ b/server-core/src/main/java/io/onedev/server/imports/ProjectImporter.java@@ -1,18 +1,20 @@package io.onedev.server.imports;import java.io.Serializable;+import java.util.List;import javax.annotation.Nullable;import io.onedev.commons.utils.TaskLogger;+import io.onedev.server.web.util.ImportStep;-public abstract class ProjectImporter<Where extends Serializable, What extends Serializable, How extends Serializable>- extends Importer<Where, What, How> {+public interface ProjectImporter extends Serializable {- private static final long serialVersionUID = 1L;+ String getName();+ List<ImportStep<? extends Serializable>> getSteps();+@Nullable- public abstract String doImport(Where where, What what, How how,- boolean dryRun, TaskLogger logger);+ String doImport(boolean dryRun, TaskLogger logger);}diff --git a/server-core/src/main/java/io/onedev/server/imports/ProjectImporterContribution.java b/server-core/src/main/java/io/onedev/server/imports/ProjectImporterContribution.javaindex a13ba0e915..ab04937b43 100644--- a/server-core/src/main/java/io/onedev/server/imports/ProjectImporterContribution.java+++ b/server-core/src/main/java/io/onedev/server/imports/ProjectImporterContribution.java@@ -1,6 +1,5 @@package io.onedev.server.imports;-import java.io.Serializable;import java.util.Collection;import io.onedev.commons.loader.ExtensionPoint;@@ -8,7 +7,7 @@ import io.onedev.commons.loader.ExtensionPoint;@ExtensionPointpublic interface ProjectImporterContribution {- Collection<ProjectImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> getImporters();+ Collection<ProjectImporter> getImporters();int getOrder();diff --git a/server-core/src/main/java/io/onedev/server/maintenance/DefaultDataManager.java b/server-core/src/main/java/io/onedev/server/maintenance/DefaultDataManager.javaindex abe655734b..b80253e449 100644--- a/server-core/src/main/java/io/onedev/server/maintenance/DefaultDataManager.java+++ b/server-core/src/main/java/io/onedev/server/maintenance/DefaultDataManager.java@@ -63,7 +63,6 @@ import io.onedev.server.persistence.annotation.Sessional;import io.onedev.server.persistence.annotation.Transactional;import io.onedev.server.ssh.SshKeyUtils;import io.onedev.server.util.init.ManualConfig;-import io.onedev.server.util.init.Skippable;import io.onedev.server.util.schedule.SchedulableTask;import io.onedev.server.util.schedule.TaskScheduler;import io.onedev.server.web.util.editablebean.NewUserBean;@@ -137,11 +136,6 @@ public class DefaultDataManager implements DataManager, Serializable {if (userManager.get(User.ROOT_ID) == null) {manualConfigs.add(new ManualConfig("Create Administrator Account", null, new NewUserBean()) {- @Override- public Skippable getSkippable() {- return null;- }-@Overridepublic void complete() {NewUserBean newUserBean = (NewUserBean) getSetting();@@ -208,11 +202,6 @@ public class DefaultDataManager implements DataManager, Serializable {manualConfigs.add(new ManualConfig("Specify System Setting", null,systemSetting, excludedProps) {- @Override- public Skippable getSkippable() {- return null;- }-@Overridepublic void complete() {settingManager.saveSystemSetting((SystemSetting) getSetting());@@ -302,11 +291,6 @@ public class DefaultDataManager implements DataManager, Serializable {manualConfigs.add(new ManualConfig("Specify Service Desk Setting", null,setting.getValue(), new HashSet<>(), true) {- @Override- public Skippable getSkippable() {- return null;- }-@Overridepublic void complete() {settingManager.saveServiceDeskSetting((ServiceDeskSetting) getSetting());@@ -329,11 +313,6 @@ public class DefaultDataManager implements DataManager, Serializable {} else if (setting.getValue() != null && !validator.validate(setting.getValue()).isEmpty()) {manualConfigs.add(new ManualConfig("Specify Mail Setting", null, setting.getValue()) {- @Override- public Skippable getSkippable() {- return null;- }-@Overridepublic void complete() {settingManager.saveMailSetting((MailSetting) getSetting());@@ -349,11 +328,6 @@ public class DefaultDataManager implements DataManager, Serializable {Serializable backupSetting = setting.getValue();manualConfigs.add(new ManualConfig("Specify Backup Setting", null, backupSetting) {- @Override- public Skippable getSkippable() {- return null;- }-@Overridepublic void complete() {settingManager.saveBackupSetting((BackupSetting) getSetting());diff --git a/server-core/src/main/java/io/onedev/server/util/CollectionUtils.java b/server-core/src/main/java/io/onedev/server/util/CollectionUtils.javaindex a52facf2bf..9acca1fb2d 100644--- a/server-core/src/main/java/io/onedev/server/util/CollectionUtils.java+++ b/server-core/src/main/java/io/onedev/server/util/CollectionUtils.java@@ -49,4 +49,15 @@ public class CollectionUtils extends org.apache.commons.collections.CollectionUtreturn list;}+ public static <K, V extends Comparable<? super V>> Map<K, V> sortByValue(Map<K, V> map) {+ List<Map.Entry<K, V>> list = new ArrayList<>(map.entrySet());+ list.sort(Map.Entry.comparingByValue());++ Map<K, V> sorted = new LinkedHashMap<>();+ for (Map.Entry<K, V> entry : list)+ sorted.put(entry.getKey(), entry.getValue());++ return sorted;+ }+}diff --git a/server-core/src/main/java/io/onedev/server/web/component/wizard/ManualConfigStep.java b/server-core/src/main/java/io/onedev/server/util/ManualConfigStep.javasimilarity index 79%rename from server-core/src/main/java/io/onedev/server/web/component/wizard/ManualConfigStep.javarename to server-core/src/main/java/io/onedev/server/util/ManualConfigStep.javaindex c162473f88..3ac5916886 100644--- a/server-core/src/main/java/io/onedev/server/web/component/wizard/ManualConfigStep.java+++ b/server-core/src/main/java/io/onedev/server/util/ManualConfigStep.java@@ -1,4 +1,4 @@-package io.onedev.server.web.component.wizard;+package io.onedev.server.util;import java.io.Serializable;@@ -8,6 +8,7 @@ import org.apache.wicket.markup.html.form.FormComponent;import io.onedev.server.util.init.ManualConfig;import io.onedev.server.web.behavior.ForceOrdinaryStyleBehavior;+import io.onedev.server.web.component.wizard.WizardStep;import io.onedev.server.web.editable.BeanContext;import io.onedev.server.web.editable.BeanEditor;@@ -29,22 +30,6 @@ public class ManualConfigStep implements WizardStep {return editor;}- @Override- public Skippable getSkippable() {- if (config.getSkippable() != null) {- return new Skippable() {-- @Override- public void skip() {- config.getSkippable().skip();- }-- };- } else {- return null;- }- }-@Overridepublic void complete() {config.complete();@@ -60,4 +45,9 @@ public class ManualConfigStep implements WizardStep {public String getDescription() {return config.getDescription();}++ @Override+ public void init() {+ }+}diff --git a/server-core/src/main/java/io/onedev/server/util/init/ManualConfig.java b/server-core/src/main/java/io/onedev/server/util/init/ManualConfig.javaindex 7e609256e9..6707dda616 100644--- a/server-core/src/main/java/io/onedev/server/util/init/ManualConfig.java+++ b/server-core/src/main/java/io/onedev/server/util/init/ManualConfig.java@@ -56,8 +56,6 @@ public abstract class ManualConfig implements Serializable {public Collection<String> getExcludeProperties() {return excludedProperties;}-- public abstract Skippable getSkippable();public abstract void complete();diff --git a/server-core/src/main/java/io/onedev/server/util/init/Skippable.java b/server-core/src/main/java/io/onedev/server/util/init/Skippable.javadeleted file mode 100644index 757e9a631d..0000000000--- a/server-core/src/main/java/io/onedev/server/util/init/Skippable.java+++ /dev/null@@ -1,5 +0,0 @@-package io.onedev.server.util.init;--public interface Skippable {- void skip();-}diff --git a/server-core/src/main/java/io/onedev/server/web/component/imports/ImportPanel.html b/server-core/src/main/java/io/onedev/server/web/component/imports/ImportPanel.htmldeleted file mode 100644index 746da03274..0000000000--- a/server-core/src/main/java/io/onedev/server/web/component/imports/ImportPanel.html+++ /dev/null@@ -1,13 +0,0 @@-<wicket:panel>- <form wicket:id="form">- <div wicket:id="retainNumbersNote" class="alert alert-notice alert-light mb-5">Importing issues- into currrent project. Please note that issue numbers will only be retained if the whole- project fork graph does not have any issues to avoid duplicate issue numbers</div>- <div wicket:id="feedback"></div>- <div wicket:id="editor"></div>- <input wicket:id="back" type="button" class="btn btn-light-primary mr-1" value="Back">- <input wicket:id="next" type="submit" class="btn btn-primary mr-1" value="Next">- <input wicket:id="dryRun" type="submit" class="btn btn-light-primary mr-1" value="Dry Run">- <input wicket:id="import" type="submit" class="btn btn-primary mr-1" value="Import">- </form>-</wicket:panel>\ No newline at end of filediff --git a/server-core/src/main/java/io/onedev/server/web/component/imports/ImportPanel.java b/server-core/src/main/java/io/onedev/server/web/component/imports/ImportPanel.javadeleted file mode 100644index b4e2237d38..0000000000--- a/server-core/src/main/java/io/onedev/server/web/component/imports/ImportPanel.java+++ /dev/null@@ -1,205 +0,0 @@-package io.onedev.server.web.component.imports;--import java.io.Serializable;-import java.util.concurrent.Callable;--import javax.annotation.Nullable;--import org.apache.wicket.ajax.AjaxRequestTarget;-import org.apache.wicket.ajax.attributes.AjaxRequestAttributes;-import org.apache.wicket.ajax.markup.html.AjaxLink;-import org.apache.wicket.ajax.markup.html.form.AjaxButton;-import org.apache.wicket.feedback.FencedFeedbackPanel;-import org.apache.wicket.markup.html.WebMarkupContainer;-import org.apache.wicket.markup.html.form.Form;-import org.apache.wicket.markup.html.panel.Panel;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--import io.onedev.commons.utils.TaskLogger;-import io.onedev.server.OneDev;-import io.onedev.server.imports.Importer;-import io.onedev.server.persistence.TransactionManager;-import io.onedev.server.web.ajaxlistener.ShowGlobalAjaxIndicatorListener;-import io.onedev.server.web.component.taskbutton.TaskButton;-import io.onedev.server.web.editable.BeanContext;-import io.onedev.server.web.page.project.ProjectPage;--@SuppressWarnings("serial")-public abstract class ImportPanel<Where extends Serializable, What extends Serializable, How extends Serializable>- extends Panel {-- private static final Logger logger = LoggerFactory.getLogger(ImportPanel.class);-- private Where where;-- private What what;-- private How how;-- public ImportPanel(String id) {- super(id);- }-- @Override- protected void onInitialize() {- super.onInitialize();-- try {- where = getImporter().getWhereClass().newInstance();- } catch (InstantiationException | IllegalAccessException e) {- throw new RuntimeException(e);- }-- Form<?> form = new Form<Void>("form");- form.add(new WebMarkupContainer("retainNumbersNote")- .setVisible(getPage() instanceof ProjectPage));- form.add(new FencedFeedbackPanel("feedback", form));- form.add(BeanContext.edit("editor", where));-- form.add(new AjaxLink<Void>("back") {-- @Override- public void onClick(AjaxRequestTarget target) {- if (how != null) {- how = null;- form.replace(BeanContext.edit("editor", what));- form.get("import").setVisible(false);- form.get("dryRun").setVisible(false);- } else {- what = null;- form.replace(BeanContext.edit("editor", where));- setVisible(false);- }-- form.get("next").setVisible(true);-- target.add(form);- }-- }.setVisible(false));-- form.add(new AjaxButton("next") {-- @Override- protected void updateAjaxAttributes(AjaxRequestAttributes attributes) {- super.updateAjaxAttributes(attributes);- attributes.getAjaxCallListeners().add(new ShowGlobalAjaxIndicatorListener());- }-- @Override- protected void onSubmit(AjaxRequestTarget target, Form<?> form) {- super.onSubmit(target, form);-- if (what == null) {- what = getImporter().getWhat(where, new TaskLogger() {-- @Override- public void log(String message, String sessionId) {- logger.info(message);- }-- });- form.replace(BeanContext.edit("editor", what));- form.get("back").setVisible(true);- } else {- how = getImporter().getHow(where, what, new TaskLogger() {-- @Override- public void log(String message, String sessionId) {- logger.info(message);- }-- });- form.replace(BeanContext.edit("editor", how));- form.get("next").setVisible(false);- form.get("import").setVisible(true);- form.get("dryRun").setVisible(true);- }-- target.add(form);- }-- @Override- protected void onError(AjaxRequestTarget target, Form<?> form) {- super.onError(target, form);- target.add(form);- }-- });-- form.add(new TaskButton("import") {-- @Override- protected void onCompleted(AjaxRequestTarget target, boolean successful) {- super.onCompleted(target, successful);-- if (successful)- onImportSuccessful(target);- }-- @Override- protected String runTask(TaskLogger logger) {- return OneDev.getInstance(TransactionManager.class).call(new Callable<String>() {-- @Override- public String call() throws Exception {- return doImport(where, what, how, false, logger);- }-- });- }-- @Override- protected String getTitle() {- return "Importing from " + getImporter().getName();- }-- @Override- protected void onError(AjaxRequestTarget target, Form<?> form) {- super.onError(target, form);- target.add(form);- }-- }.setVisible(false));-- form.add(new TaskButton("dryRun") {-- @Override- protected String runTask(TaskLogger logger) {- return OneDev.getInstance(TransactionManager.class).call(new Callable<String>() {-- @Override- public String call() throws Exception {- return doImport(where, what, how, true, logger);- }-- });- }-- @Override- protected String getTitle() {- return "Test importing from " + getImporter().getName();- }-- @Override- protected void onError(AjaxRequestTarget target, Form<?> form) {- super.onError(target, form);- target.add(form);- }-- }.setVisible(false));-- form.setOutputMarkupId(true);- add(form);- }-- protected abstract Importer<Where, What, How> getImporter();-- @Nullable- protected abstract String doImport(Where where, What what, How how,- boolean dryRun, TaskLogger logger);-- protected abstract void onImportSuccessful(AjaxRequestTarget target);--}diff --git a/server-core/src/main/java/io/onedev/server/web/component/issue/list/IssueListPanel.java b/server-core/src/main/java/io/onedev/server/web/component/issue/list/IssueListPanel.javaindex cb40f604f2..c3b2cd678e 100644--- a/server-core/src/main/java/io/onedev/server/web/component/issue/list/IssueListPanel.java+++ b/server-core/src/main/java/io/onedev/server/web/component/issue/list/IssueListPanel.java@@ -1,6 +1,5 @@package io.onedev.server.web.component.issue.list;-import java.io.Serializable;import java.util.ArrayList;import java.util.Collection;import java.util.Comparator;@@ -324,7 +323,7 @@ public abstract class IssueListPanel extends Panel {@Overrideprotected List<MenuItem> getMenuItems(FloatingPanel dropdown) {- Collection<IssueImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> importers = new ArrayList<>();+ Collection<IssueImporter> importers = new ArrayList<>();List<IssueImporterContribution> contributions =new ArrayList<>(OneDev.getExtensions(IssueImporterContribution.class));@@ -341,7 +340,7 @@ public abstract class IssueListPanel extends Panel {importers.addAll(contribution.getImporters());List<MenuItem> menuItems = new ArrayList<>();- for (IssueImporter<? extends Serializable, ? extends Serializable, ? extends Serializable> importer: importers) {+ for (IssueImporter importer: importers) {menuItems.add(new MenuItem() {@Overridediff --git a/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.java b/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.javaindex c03ddeae96..4d8b13cf14 100644--- a/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.java+++ b/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.java@@ -1,7 +1,6 @@package io.onedev.server.web.component.project.list;import java.io.IOException;-import java.io.Serializable;import java.util.ArrayList;import java.util.Collection;import java.util.Collections;@@ -894,7 +893,7 @@ public class ProjectListPanel extends Panel {@Overrideprotected List<MenuItem> getMenuItems(FloatingPanel dropdown) {- Collection<ProjectImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> importers = new ArrayList<>();+ Collection<ProjectImporter> importers = new ArrayList<>();List<ProjectImporterContribution> contributions =new ArrayList<>(OneDev.getExtensions(ProjectImporterContribution.class));Collections.sort(contributions, new Comparator<ProjectImporterContribution>() {@@ -910,7 +909,7 @@ public class ProjectListPanel extends Panel {importers.addAll(contribution.getImporters());List<MenuItem> menuItems = new ArrayList<>();- for (ProjectImporter<? extends Serializable, ? extends Serializable, ? extends Serializable> importer: importers) {+ for (ProjectImporter importer: importers) {menuItems.add(new MenuItem() {@Overridediff --git a/server-core/src/main/java/io/onedev/server/web/component/wizard/DefaultEndActionsPanel.html b/server-core/src/main/java/io/onedev/server/web/component/wizard/DefaultEndActionsPanel.htmlnew file mode 100644index 0000000000..6552445db6--- /dev/null+++ b/server-core/src/main/java/io/onedev/server/web/component/wizard/DefaultEndActionsPanel.html@@ -0,0 +1,3 @@+<wicket:panel>+ <button type="submit" wicket:id="finish" class="btn btn-primary finish">Finish</button>+</wicket:panel>diff --git a/server-core/src/main/java/io/onedev/server/web/component/wizard/DefaultEndActionsPanel.java b/server-core/src/main/java/io/onedev/server/web/component/wizard/DefaultEndActionsPanel.javanew file mode 100644index 0000000000..cd5df3414a--- /dev/null+++ b/server-core/src/main/java/io/onedev/server/web/component/wizard/DefaultEndActionsPanel.java@@ -0,0 +1,46 @@+package io.onedev.server.web.component.wizard;++import org.apache.wicket.markup.html.form.Button;+import org.apache.wicket.markup.html.panel.Panel;++import io.onedev.server.OneDev;+import io.onedev.server.persistence.TransactionManager;++@SuppressWarnings("serial")+public abstract class DefaultEndActionsPanel extends Panel {++ private final WizardPanel wizard;++ public DefaultEndActionsPanel(String id, WizardPanel wizard) {+ super(id);+ this.wizard = wizard;+ }++ @Override+ protected void onInitialize() {+ super.onInitialize();++ add(new Button("finish") {++ @Override+ public void onSubmit() {+ super.onSubmit();++ OneDev.getInstance(TransactionManager.class).run(new Runnable() {++ @Override+ public void run() {+ wizard.getActiveStep().complete();+ }++ });+ finished();+ }++ });++ }++ protected abstract void finished();++}diff --git a/server-core/src/main/java/io/onedev/server/web/component/wizard/Skippable.java b/server-core/src/main/java/io/onedev/server/web/component/wizard/Skippable.javadeleted file mode 100644index 6ebdd1e96c..0000000000--- a/server-core/src/main/java/io/onedev/server/web/component/wizard/Skippable.java+++ /dev/null@@ -1,5 +0,0 @@-package io.onedev.server.web.component.wizard;--public interface Skippable {- void skip();-}\ No newline at end of filediff --git a/server-core/src/main/java/io/onedev/server/web/component/wizard/Wizard.html b/server-core/src/main/java/io/onedev/server/web/component/wizard/WizardPanel.htmlsimilarity index 74%rename from server-core/src/main/java/io/onedev/server/web/component/wizard/Wizard.htmlrename to server-core/src/main/java/io/onedev/server/web/component/wizard/WizardPanel.htmlindex 8256951924..df61bd27f6 100644--- a/server-core/src/main/java/io/onedev/server/web/component/wizard/Wizard.html+++ b/server-core/src/main/java/io/onedev/server/web/component/wizard/WizardPanel.html@@ -10,9 +10,8 @@</div><div class="foot"><button wicket:id="previous" type="button" class="btn btn-light-primary previous">Previous</button>- <button wicket:id="skip" type="button" class="btn btn-light-primary skip">Skip</button><button type="submit" wicket:id="next" class="btn btn-primary next">Next</button>- <button type="submit" wicket:id="finish" class="btn btn-primary finish">Finish</button>+ <span wicket:id="endActions"></span></div></form></wicket:panel>diff --git a/server-core/src/main/java/io/onedev/server/web/component/wizard/Wizard.java b/server-core/src/main/java/io/onedev/server/web/component/wizard/WizardPanel.javasimilarity index 75%rename from server-core/src/main/java/io/onedev/server/web/component/wizard/Wizard.javarename to server-core/src/main/java/io/onedev/server/web/component/wizard/WizardPanel.javaindex dcb96cc070..7b896873e2 100644--- a/server-core/src/main/java/io/onedev/server/web/component/wizard/Wizard.java+++ b/server-core/src/main/java/io/onedev/server/web/component/wizard/WizardPanel.java@@ -10,10 +10,13 @@ package io.onedev.server.web.component.wizard;import java.util.List;+import org.apache.wicket.Component;+import org.apache.wicket.behavior.Behavior;import org.apache.wicket.feedback.FencedFeedbackPanel;import org.apache.wicket.markup.ComponentTag;import org.apache.wicket.markup.head.CssHeaderItem;import org.apache.wicket.markup.head.IHeaderResponse;+import org.apache.wicket.markup.html.WebMarkupContainer;import org.apache.wicket.markup.html.basic.Label;import org.apache.wicket.markup.html.form.Button;import org.apache.wicket.markup.html.form.Form;@@ -27,13 +30,13 @@ import io.onedev.server.OneDev;import io.onedev.server.persistence.TransactionManager;@SuppressWarnings("serial")-public abstract class Wizard extends Panel {+public abstract class WizardPanel extends Panel {private List<? extends WizardStep> steps;private int activeStepIndex;- public Wizard(String id, List<? extends WizardStep> steps) {+ public WizardPanel(String id, List<? extends WizardStep> steps) {super(id);Preconditions.checkArgument(steps != null && !steps.isEmpty());@@ -72,6 +75,7 @@ public abstract class Wizard extends Panel {});form.add(new FencedFeedbackPanel("feedback", form));+ getActiveStep().init();form.add(getActiveStep().render("content"));form.add(new Link<Void>("previous") {@@ -95,24 +99,6 @@ public abstract class Wizard extends Panel {tag.append("class", "disabled", " ");}- });- form.add(new Link<Void>("skip") {-- @Override- public void onClick() {- getActiveStep().getSkippable().skip();- if (activeStepIndex == steps.size() - 1)- finished();- else- form.replace(getActiveStep().render("content"));- }-- @Override- protected void onConfigure() {- super.onConfigure();- setVisible(getActiveStep().getSkippable() != null);- }-});form.add(new Button("next") {@@ -128,6 +114,7 @@ public abstract class Wizard extends Panel {});activeStepIndex++;+ getActiveStep().init();form.replace(getActiveStep().render("content"));}@@ -138,42 +125,29 @@ public abstract class Wizard extends Panel {}});- form.add(new Button("finish") {++ form.add(newEndActions("endActions").add(new Behavior() {@Override- public void onSubmit() {- super.onSubmit();- OneDev.getInstance(TransactionManager.class).run(new Runnable() {-- @Override- public void run() {- getActiveStep().complete();- }-- });- finished();+ public void onConfigure(Component component) {+ super.onConfigure(component);+ component.setVisible(activeStepIndex == steps.size()-1);}- @Override- protected void onConfigure() {- super.onConfigure();- setVisible(activeStepIndex == steps.size()-1);- }-- });+ }));add(form);}+ protected abstract WebMarkupContainer newEndActions(String componentId);+@Overridepublic void renderHead(IHeaderResponse response) {super.renderHead(response);response.render(CssHeaderItem.forReference(new WizardResourceReference()));}- private WizardStep getActiveStep() {+ public WizardStep getActiveStep() {return steps.get(activeStepIndex);}- protected abstract void finished();-}diff --git a/server-core/src/main/java/io/onedev/server/web/component/wizard/WizardStep.java b/server-core/src/main/java/io/onedev/server/web/component/wizard/WizardStep.javaindex 3fdf07bb60..91dd046d53 100644--- a/server-core/src/main/java/io/onedev/server/web/component/wizard/WizardStep.java+++ b/server-core/src/main/java/io/onedev/server/web/component/wizard/WizardStep.java@@ -22,7 +22,7 @@ public interface WizardStep extends Serializable {@NullableString getDescription();- Skippable getSkippable();+ void init();void complete();diff --git a/server-core/src/main/java/io/onedev/server/web/component/wizard/wizard.css b/server-core/src/main/java/io/onedev/server/web/component/wizard/wizard.cssindex cc626c7ccf..84f58a913d 100644--- a/server-core/src/main/java/io/onedev/server/web/component/wizard/wizard.css+++ b/server-core/src/main/java/io/onedev/server/web/component/wizard/wizard.css@@ -7,7 +7,7 @@.wizard>.body {margin-bottom: 1.5rem;}-.wizard>.foot>* {+.wizard>.foot button {margin-right: 1rem;}.wizard>.foot>:last-child {diff --git a/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImportPage.html b/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImportPage.htmlindex 315da2b63f..2c5e651c56 100644--- a/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImportPage.html+++ b/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImportPage.html@@ -1,5 +1,11 @@<wicket:extend><div class="project-import card m-2 m-sm-5">- <div wicket:id="importer" class="card-body"></div>+ <div class="card-body">+ <div wicket:id="wizard"></div>+ </div></div>+ <wicket:fragment wicket:id="endActionsFrag">+ <input wicket:id="dryRun" type="submit" class="btn btn-light-primary mr-1" value="Dry Run">+ <input wicket:id="import" type="submit" class="btn btn-primary mr-1" value="Import">+ </wicket:fragment></wicket:extend>diff --git a/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImportPage.java b/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImportPage.javaindex ae2f5e24b9..468319638a 100644--- a/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImportPage.java+++ b/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImportPage.java@@ -1,45 +1,48 @@package io.onedev.server.web.page.project.imports;-import java.io.Serializable;+import java.util.concurrent.Callable;import org.apache.wicket.Component;import org.apache.wicket.RestartResponseException;import org.apache.wicket.ajax.AjaxRequestTarget;+import org.apache.wicket.markup.html.WebMarkupContainer;import org.apache.wicket.markup.html.basic.Label;+import org.apache.wicket.markup.html.form.Form;+import org.apache.wicket.markup.html.panel.Fragment;import org.apache.wicket.request.mapper.parameter.PageParameters;import com.google.common.collect.Lists;import io.onedev.commons.utils.TaskLogger;import io.onedev.server.OneDev;-import io.onedev.server.imports.Importer;import io.onedev.server.imports.ProjectImporter;import io.onedev.server.imports.ProjectImporterContribution;import io.onedev.server.model.Project;+import io.onedev.server.persistence.TransactionManager;import io.onedev.server.search.entity.EntitySort;import io.onedev.server.search.entity.EntitySort.Direction;import io.onedev.server.search.entity.project.ProjectQuery;import io.onedev.server.security.SecurityUtils;-import io.onedev.server.web.component.imports.ImportPanel;+import io.onedev.server.web.component.taskbutton.TaskButton;+import io.onedev.server.web.component.wizard.WizardPanel;import io.onedev.server.web.page.layout.LayoutPage;import io.onedev.server.web.page.project.ProjectListPage;@SuppressWarnings("serial")-public class ProjectImportPage<Where extends Serializable, What extends Serializable, How extends Serializable> extends LayoutPage {+public class ProjectImportPage extends LayoutPage {private static final String PARAM_IMPORTER = "importer";- private ProjectImporter<Where, What, How> importer;+ private ProjectImporter importer;- @SuppressWarnings("unchecked")public ProjectImportPage(PageParameters params) {super(params);String importerName = params.get(PARAM_IMPORTER).toString();for (ProjectImporterContribution contribution: OneDev.getExtensions(ProjectImporterContribution.class)) {- for (ProjectImporter<? extends Serializable, ? extends Serializable, ? extends Serializable> importer: contribution.getImporters()) {+ for (ProjectImporter importer: contribution.getImporters()) {if (importer.getName().equals(importerName)) {- this.importer = (ProjectImporter<Where, What, How>) importer;+ this.importer = importer;break;}}@@ -53,27 +56,81 @@ public class ProjectImportPage<Where extends Serializable, What extends Serializprotected void onInitialize() {super.onInitialize();- add(new ImportPanel<Where, What, How>("importer") {+ add(new WizardPanel("wizard", importer.getSteps()) {@Override- protected Importer<Where, What, How> getImporter() {- return importer;- }+ protected WebMarkupContainer newEndActions(String componentId) {+ Fragment fragment = new Fragment(componentId, "endActionsFrag", ProjectImportPage.this);++ fragment.add(new TaskButton("import") {- @Override- protected String doImport(Where where, What what, How how,- boolean dryRun, TaskLogger logger) {- return importer.doImport(where, what, how, dryRun, logger);- }+ @Override+ protected void onCompleted(AjaxRequestTarget target, boolean successful) {+ super.onCompleted(target, successful);- @Override- protected void onImportSuccessful(AjaxRequestTarget target) {- EntitySort sort = new EntitySort();- sort.setField(Project.NAME_UPDATE_DATE);- sort.setDirection(Direction.DESCENDING);- ProjectQuery query = new ProjectQuery(null, Lists.newArrayList(sort));- PageParameters params = ProjectListPage.paramsOf(query.toString(), 0, 0);- throw new RestartResponseException(ProjectListPage.class, params);+ if (successful) {+ EntitySort sort = new EntitySort();+ sort.setField(Project.NAME_UPDATE_DATE);+ sort.setDirection(Direction.DESCENDING);+ ProjectQuery query = new ProjectQuery(null, Lists.newArrayList(sort));+ PageParameters params = ProjectListPage.paramsOf(query.toString(), 0, 0);+ throw new RestartResponseException(ProjectListPage.class, params);+ }+ }++ @Override+ protected String runTask(TaskLogger logger) {+ return OneDev.getInstance(TransactionManager.class).call(new Callable<String>() {++ @Override+ public String call() throws Exception {+ return importer.doImport(false, logger);+ }++ });+ }++ @Override+ protected String getTitle() {+ return "Importing from " + importer.getName();+ }++ @Override+ protected void onError(AjaxRequestTarget target, Form<?> form) {+ super.onError(target, form);+ target.add(form);+ }++ });++ fragment.add(new TaskButton("dryRun") {++ @Override+ protected String runTask(TaskLogger logger) {+ return OneDev.getInstance(TransactionManager.class).call(new Callable<String>() {++ @Override+ public String call() throws Exception {+ return importer.doImport(true, logger);+ }++ });+ }++ @Override+ protected String getTitle() {+ return "Test importing from " + importer.getName();+ }++ @Override+ protected void onError(AjaxRequestTarget target, Form<?> form) {+ super.onError(target, form);+ target.add(form);+ }++ });++ return fragment;}});diff --git a/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.html b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.htmlindex 01bcc25067..89e6b80468 100644--- a/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.html+++ b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.html@@ -1,5 +1,14 @@<wicket:extend><div class="issue-import card m-2 m-sm-5">- <div wicket:id="importer" class="card-body"></div>+ <div class="card-body">+ <div class="alert alert-notice alert-light mb-5">Importing issues+ into currrent project. Please note that issue numbers will only be retained if the whole+ project fork graph does not have any issues to avoid duplicate issue numbers</div>+ <div wicket:id="wizard"></div>+ </div></div>+ <wicket:fragment wicket:id="endActionsFrag">+ <input wicket:id="dryRun" type="submit" class="btn btn-light-primary mr-1" value="Dry Run">+ <input wicket:id="import" type="submit" class="btn btn-primary mr-1" value="Import">+ </wicket:fragment></wicket:extend>diff --git a/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.java b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.javaindex 79d1facc84..ec4eed83ee 100644--- a/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.java+++ b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.java@@ -1,11 +1,15 @@package io.onedev.server.web.page.project.issues.imports;import java.io.Serializable;+import java.util.concurrent.Callable;import org.apache.wicket.Component;import org.apache.wicket.RestartResponseException;import org.apache.wicket.ajax.AjaxRequestTarget;+import org.apache.wicket.markup.html.WebMarkupContainer;import org.apache.wicket.markup.html.basic.Label;+import org.apache.wicket.markup.html.form.Form;+import org.apache.wicket.markup.html.panel.Fragment;import org.apache.wicket.request.mapper.parameter.PageParameters;import com.google.common.collect.Lists;@@ -13,17 +17,17 @@ import com.google.common.collect.Lists;import io.onedev.commons.utils.TaskLogger;import io.onedev.server.OneDev;import io.onedev.server.entitymanager.IssueManager;-import io.onedev.server.entitymanager.ProjectManager;-import io.onedev.server.imports.Importer;import io.onedev.server.imports.IssueImporter;import io.onedev.server.imports.IssueImporterContribution;import io.onedev.server.model.Issue;import io.onedev.server.model.Project;+import io.onedev.server.persistence.TransactionManager;import io.onedev.server.search.entity.EntitySort;import io.onedev.server.search.entity.EntitySort.Direction;import io.onedev.server.search.entity.issue.IssueQuery;import io.onedev.server.security.SecurityUtils;-import io.onedev.server.web.component.imports.ImportPanel;+import io.onedev.server.web.component.taskbutton.TaskButton;+import io.onedev.server.web.component.wizard.WizardPanel;import io.onedev.server.web.page.project.ProjectPage;import io.onedev.server.web.page.project.dashboard.ProjectDashboardPage;import io.onedev.server.web.page.project.issues.list.ProjectIssueListPage;@@ -33,17 +37,16 @@ public class IssueImportPage<Where extends Serializable, What extends Serializabprivate static final String PARAM_IMPORTER = "importer";- private IssueImporter<Where, What, How> importer;+ private IssueImporter importer;- @SuppressWarnings("unchecked")public IssueImportPage(PageParameters params) {super(params);String importerName = params.get(PARAM_IMPORTER).toString();for (IssueImporterContribution contribution: OneDev.getExtensions(IssueImporterContribution.class)) {- for (IssueImporter<? extends Serializable, ? extends Serializable, ? extends Serializable> importer: contribution.getImporters()) {+ for (IssueImporter importer: contribution.getImporters()) {if (importer.getName().equals(importerName)) {- this.importer = (IssueImporter<Where, What, How>) importer;+ this.importer = importer;break;}}@@ -57,43 +60,100 @@ public class IssueImportPage<Where extends Serializable, What extends Serializabprotected void onInitialize() {super.onInitialize();- Long projectId = getProject().getId();- add(new ImportPanel<Where, What, How>("importer") {+ add(new WizardPanel("wizard", importer.getSteps()) {@Override- protected Importer<Where, What, How> getImporter() {- return importer;- }+ protected WebMarkupContainer newEndActions(String componentId) {+ Fragment fragment = new Fragment(componentId, "endActionsFrag", IssueImportPage.this);++ fragment.add(new TaskButton("import") {- @Override- protected String doImport(Where where, What what, How how, boolean dryRun, TaskLogger logger) {- IssueManager issueManager = OneDev.getInstance(IssueManager.class);- Project project = OneDev.getInstance(ProjectManager.class).load(projectId);- Project numberScope = project.getForkRoot();- try {- issueManager.resetNextNumber(numberScope);- Long nextNumber = issueManager.getNextNumber(numberScope);- issueManager.resetNextNumber(numberScope);- return importer.doImport(where, what, how, project, nextNumber==1L, dryRun, logger);- } finally {- issueManager.resetNextNumber(numberScope);- }- }+ @Override+ protected void onCompleted(AjaxRequestTarget target, boolean successful) {+ super.onCompleted(target, successful);- @Override- protected void onImportSuccessful(AjaxRequestTarget target) {- EntitySort sort = new EntitySort();- sort.setField(Issue.NAME_NUMBER);- sort.setDirection(Direction.DESCENDING);- IssueQuery query = new IssueQuery(null, Lists.newArrayList(sort));+ if (successful) {+ EntitySort sort = new EntitySort();+ sort.setField(Issue.NAME_NUMBER);+ sort.setDirection(Direction.DESCENDING);+ IssueQuery query = new IssueQuery(null, Lists.newArrayList(sort));++ PageParameters params = ProjectIssueListPage.paramsOf(getProject(), query.toString(), 0);+ throw new RestartResponseException(ProjectIssueListPage.class, params);+ }+ }++ @Override+ protected String runTask(TaskLogger logger) {+ return OneDev.getInstance(TransactionManager.class).call(new Callable<String>() {++ @Override+ public String call() throws Exception {+ return doImport(false, logger);+ }++ });+ }++ @Override+ protected String getTitle() {+ return "Importing from " + importer.getName();+ }++ @Override+ protected void onError(AjaxRequestTarget target, Form<?> form) {+ super.onError(target, form);+ target.add(form);+ }++ });- PageParameters params = ProjectIssueListPage.paramsOf(getProject(), query.toString(), 0);- throw new RestartResponseException(ProjectIssueListPage.class, params);+ fragment.add(new TaskButton("dryRun") {++ @Override+ protected String runTask(TaskLogger logger) {+ return OneDev.getInstance(TransactionManager.class).call(new Callable<String>() {++ @Override+ public String call() throws Exception {+ return doImport(true, logger);+ }++ });+ }++ @Override+ protected String getTitle() {+ return "Test importing from " + importer.getName();+ }++ @Override+ protected void onError(AjaxRequestTarget target, Form<?> form) {+ super.onError(target, form);+ target.add(form);+ }++ });++ return fragment;}});}++ private String doImport(boolean dryRun, TaskLogger logger) {+ IssueManager issueManager = OneDev.getInstance(IssueManager.class);+ Project numberScope = getProject().getForkRoot();+ try {+ issueManager.resetNextNumber(numberScope);+ Long nextNumber = issueManager.getNextNumber(numberScope);+ issueManager.resetNextNumber(numberScope);+ return importer.doImport(getProject(), nextNumber==1L, dryRun, logger);+ } finally {+ issueManager.resetNextNumber(numberScope);+ }+ }@Overrideprotected boolean isPermitted() {diff --git a/server-core/src/main/java/io/onedev/server/web/page/simple/serverinit/ServerInitPage.java b/server-core/src/main/java/io/onedev/server/web/page/simple/serverinit/ServerInitPage.javaindex 19e27a0dd3..d44952cd28 100644--- a/server-core/src/main/java/io/onedev/server/web/page/simple/serverinit/ServerInitPage.java+++ b/server-core/src/main/java/io/onedev/server/web/page/simple/serverinit/ServerInitPage.java@@ -14,12 +14,12 @@ import io.onedev.server.OneDev;import io.onedev.server.entitymanager.UserManager;import io.onedev.server.model.User;import io.onedev.server.security.SecurityUtils;+import io.onedev.server.util.ManualConfigStep;import io.onedev.server.util.init.InitStage;import io.onedev.server.util.init.ManualConfig;-import io.onedev.server.util.init.Skippable;import io.onedev.server.web.WebSession;-import io.onedev.server.web.component.wizard.ManualConfigStep;-import io.onedev.server.web.component.wizard.Wizard;+import io.onedev.server.web.component.wizard.DefaultEndActionsPanel;+import io.onedev.server.web.component.wizard.WizardPanel;import io.onedev.server.web.page.project.ProjectListPage;import io.onedev.server.web.page.simple.SimplePage;@@ -43,26 +43,6 @@ public class ServerInitPage extends SimplePage {clonedConfigs.add(new ManualConfig(lastConfig.getTitle(), lastConfig.getDescription(),lastConfig.getSetting(), lastConfig.getExcludeProperties(), lastConfig.isForceOrdinaryStyle()) {- @Override- public Skippable getSkippable() {- final Skippable skippable = lastConfig.getSkippable();- if (skippable != null) {- return new Skippable() {-- @Override- public void skip() {- skippable.skip();- InitStage initStage = OneDev.getInstance().getInitStage();- if (initStage != null)- initStage.finished();- }-- };- } else {- return null;- }- }-@Overridepublic void complete() {lastConfig.complete();@@ -92,15 +72,23 @@ public class ServerInitPage extends SimplePage {List<ManualConfigStep> configSteps = new ArrayList<ManualConfigStep>();for (ManualConfig each: initStage.getManualConfigs())configSteps.add(new ManualConfigStep(each));- add(new Wizard("wizard", configSteps) {+ add(new WizardPanel("wizard", configSteps) {@Override- protected void finished() {- WebSession.get().logout();- User root = OneDev.getInstance(UserManager.class).getRoot();- SecurityUtils.getSubject().runAs(root.getPrincipals());- throw new RestartResponseException(ProjectListPage.class);+ protected WebMarkupContainer newEndActions(String componentId) {+ return new DefaultEndActionsPanel(componentId, this) {++ @Override+ protected void finished() {+ WebSession.get().logout();+ User root = OneDev.getInstance(UserManager.class).getRoot();+ SecurityUtils.getSubject().runAs(root.getPrincipals());+ throw new RestartResponseException(ProjectListPage.class);+ }++ };}+});} else {diff --git a/server-core/src/main/java/io/onedev/server/web/util/ImportStep.java b/server-core/src/main/java/io/onedev/server/web/util/ImportStep.javanew file mode 100644index 0000000000..82a1f37633--- /dev/null+++ b/server-core/src/main/java/io/onedev/server/web/util/ImportStep.java@@ -0,0 +1,42 @@+package io.onedev.server.web.util;++import java.io.Serializable;++import org.apache.wicket.Component;++import io.onedev.server.web.component.wizard.WizardStep;+import io.onedev.server.web.editable.BeanContext;++public abstract class ImportStep<T extends Serializable> implements WizardStep {++ private static final long serialVersionUID = 1L;++ private T setting;++ @Override+ public Component render(String componentId) {+ return BeanContext.edit(componentId, setting);+ }++ @Override+ public String getDescription() {+ return null;+ }++ protected abstract T newSetting();++ @Override+ public void init() {+ setting = newSetting();+ }++ @Override+ public void complete() {++ }++ public T getSetting() {+ return setting;+ }++}diff --git a/server-plugin/pom.xml b/server-plugin/pom.xmlindex 35312c8259..75e39b2f41 100644--- a/server-plugin/pom.xml+++ b/server-plugin/pom.xml@@ -1,5 +1,7 @@<?xml version="1.0" encoding="UTF-8"?>-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">+<project xmlns="http://maven.apache.org/POM/4.0.0"+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"><modelVersion>4.0.0</modelVersion><artifactId>server-plugin</artifactId><packaging>pom</packaging>@@ -20,31 +22,31 @@<module>server-plugin-executor-kubernetes</module><module>server-plugin-executor-serverdocker</module><module>server-plugin-buildspec-maven</module>- <module>server-plugin-buildspec-gradle</module>- <module>server-plugin-buildspec-node</module>- <module>server-plugin-authenticator-ldap</module>- <module>server-plugin-sso-openid</module>- <module>server-plugin-report-markdown</module>- <module>server-plugin-report-jest</module>- <module>server-plugin-report-clover</module>- <module>server-plugin-report-checkstyle</module>- <module>server-plugin-import-github</module>- <module>server-plugin-import-youtrack</module>- <module>server-plugin-import-gitlab</module>- <module>server-plugin-import-gitea</module>- <module>server-plugin-import-bitbucketcloud</module>- <module>server-plugin-import-jiracloud</module>- <module>server-plugin-executor-remotedocker</module>- <module>server-plugin-executor-servershell</module>- <module>server-plugin-executor-remoteshell</module>- <module>server-plugin-report-unittest</module>- <module>server-plugin-report-junit</module>- <module>server-plugin-report-coverage</module>- <module>server-plugin-report-jacoco</module>- <module>server-plugin-report-problem</module>- <module>server-plugin-report-spotbugs</module>- <module>server-plugin-report-pmd</module>- <module>server-plugin-report-cpd</module>- <module>server-plugin-sso-discord</module>- </modules>+ <module>server-plugin-buildspec-gradle</module>+ <module>server-plugin-buildspec-node</module>+ <module>server-plugin-authenticator-ldap</module>+ <module>server-plugin-sso-openid</module>+ <module>server-plugin-report-markdown</module>+ <module>server-plugin-report-jest</module>+ <module>server-plugin-report-clover</module>+ <module>server-plugin-report-checkstyle</module>+ <module>server-plugin-import-github</module>+ <module>server-plugin-import-gitlab</module>+ <module>server-plugin-import-gitea</module>+ <module>server-plugin-import-bitbucketcloud</module>+ <module>server-plugin-import-jiracloud</module>+ <module>server-plugin-import-youtrack</module>+ <module>server-plugin-executor-remotedocker</module>+ <module>server-plugin-executor-servershell</module>+ <module>server-plugin-executor-remoteshell</module>+ <module>server-plugin-report-unittest</module>+ <module>server-plugin-report-junit</module>+ <module>server-plugin-report-coverage</module>+ <module>server-plugin-report-jacoco</module>+ <module>server-plugin-report-problem</module>+ <module>server-plugin-report-spotbugs</module>+ <module>server-plugin-report-pmd</module>+ <module>server-plugin-report-cpd</module>+ <module>server-plugin-sso-discord</module>+ </modules></project>\ No newline at end of filediff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketPluginModule.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketPluginModule.javaindex 82b42ed1bb..9395354c07 100644--- a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketPluginModule.java+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketPluginModule.java@@ -1,6 +1,5 @@package io.onedev.server.plugin.imports.bitbucketcloud;-import java.io.Serializable;import java.util.Collection;import com.google.common.collect.Lists;@@ -15,6 +14,9 @@ import io.onedev.server.imports.ProjectImporterContribution;*/public class BitbucketPluginModule extends AbstractPluginModule {++ static final String NAME = "Bitbucket Cloud";+@Overrideprotected void configure() {super.configure();@@ -23,7 +25,7 @@ public class BitbucketPluginModule extends AbstractPluginModule {contribute(ProjectImporterContribution.class, new ProjectImporterContribution() {@Override- public Collection<ProjectImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> getImporters() {+ public Collection<ProjectImporter> getImporters() {return Lists.newArrayList(new BitbucketProjectImporter());}diff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketProjectImporter.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketProjectImporter.javaindex 5542851687..753b3009e8 100644--- a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketProjectImporter.java+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketProjectImporter.java@@ -1,106 +1,108 @@package io.onedev.server.plugin.imports.bitbucketcloud;-import java.util.ArrayList;-import java.util.Collection;+import java.io.Serializable;+import java.util.List;-import javax.ws.rs.client.Client;+import com.google.common.collect.Lists;-import org.apache.http.client.utils.URIBuilder;--import com.fasterxml.jackson.databind.JsonNode;-import com.google.common.base.Preconditions;--import io.onedev.commons.utils.ExplicitException;import io.onedev.commons.utils.TaskLogger;-import io.onedev.server.OneDev;-import io.onedev.server.entitymanager.ProjectManager;import io.onedev.server.imports.ProjectImporter;-import io.onedev.server.model.Project;-import io.onedev.server.storage.StorageManager;-import io.onedev.server.util.JerseyUtils;+import io.onedev.server.web.util.ImportStep;-public class BitbucketProjectImporter extends ProjectImporter<ImportServer, ProjectImportSource, ProjectImportOption> {+public class BitbucketProjectImporter implements ProjectImporter {private static final long serialVersionUID = 1L;- @Override- public String getName() {- return ImportUtils.NAME;- }+ private final ImportStep<ImportServer> serverStep = new ImportStep<ImportServer>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Authenticate to Bitbucket Cloud";+ }++ @Override+ protected ImportServer newSetting() {+ return new ImportServer();+ }++ };- @Override- public String doImport(ImportServer where, ProjectImportSource what, ProjectImportOption how,- boolean dryRun, TaskLogger logger) {- Collection<Long> projectIds = new ArrayList<>();- Client client = where.newClient();- try {- for (ProjectMapping projectMapping: what.getProjectMappings()) {- logger.log("Cloning code from repository " + projectMapping.getBitbucketRepo() + "...");-- String apiEndpoint = where.getApiEndpoint("/repositories/" + projectMapping.getBitbucketRepo());- JsonNode repoNode = JerseyUtils.get(client, apiEndpoint, logger);- ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);- Project project = projectManager.initialize(projectMapping.getOneDevProject());- Preconditions.checkState(project.isNew());-- project.setDescription(repoNode.get("description").asText(null));-- boolean isPrivate = repoNode.get("is_private").asBoolean();- if (!isPrivate && how.getPublicRole() != null)- project.setDefaultRole(how.getPublicRole());-- String cloneUrl = null;- for (JsonNode cloneNode: repoNode.get("links").get("clone")) {- if (cloneNode.get("name").asText().equals("https")) {- cloneUrl = cloneNode.get("href").asText();- break;- }- }- if (cloneUrl == null)- throw new ExplicitException("Https clone url not found");-- URIBuilder builder = new URIBuilder(cloneUrl);- if (isPrivate)- builder.setUserInfo(where.getUserName(), where.getAppPassword());-- if (!dryRun) {- projectManager.clone(project, builder.build().toString());- projectIds.add(project.getId());- }+ private final ImportStep<ImportWorkspace> workspaceStep = new ImportStep<ImportWorkspace>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Choose workspace";+ }++ @Override+ protected ImportWorkspace newSetting() {+ ImportWorkspace workspace = new ImportWorkspace();+ workspace.server = serverStep.getSetting();+ return workspace;+ }++ };++ private final ImportStep<ImportRepositories> repositoriesStep = new ImportStep<ImportRepositories>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Specify repositories";+ }++ @Override+ protected ImportRepositories newSetting() {+ ImportRepositories repositories = new ImportRepositories();+ String workspace = workspaceStep.getSetting().getWorkspace();+ for (String repository: serverStep.getSetting().listRepositories(workspace)) {+ ProjectMapping projectMapping = new ProjectMapping();+ projectMapping.setBitbucketRepo(repository);+ projectMapping.setOneDevProject(repository);+ repositories.getProjectMappings().add(projectMapping);}- return "Repositories imported successfully";- } catch (Exception e) {- for (Long projectId: projectIds)- OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);- throw new RuntimeException(e);- } finally {- client.close();+ return repositories;}- }++ };++ private final ImportStep<ImportOption> optionStep = new ImportStep<ImportOption>() {- @Override- public ProjectImportSource getWhat(ImportServer where, TaskLogger logger) {- ProjectImportSource importSource = new ProjectImportSource();- Client client = where.newClient();- try {- String apiEndpoint = where.getApiEndpoint("/repositories?role=member");- for (JsonNode repoNode: ImportUtils.list(client, apiEndpoint, logger)) {- String fullName = repoNode.get("full_name").asText();- ProjectMapping projectMapping = new ProjectMapping();- projectMapping.setBitbucketRepo(fullName);- projectMapping.setOneDevProject(fullName);- importSource.getProjectMappings().add(projectMapping);- }- } finally {- client.close();+ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Specify import option";+ }++ @Override+ protected ImportOption newSetting() {+ return new ImportOption();}- return importSource;++ };++ @Override+ public String getName() {+ return BitbucketPluginModule.NAME;+ }++ @Override+ public String doImport(boolean dryRun, TaskLogger logger) {+ ImportRepositories repositories = repositoriesStep.getSetting();+ ImportOption option = optionStep.getSetting();+ return serverStep.getSetting().doImport(repositories, option, dryRun, logger);}@Override- public ProjectImportOption getHow(ImportServer where, ProjectImportSource what, TaskLogger logger) {- return new ProjectImportOption();+ public List<ImportStep<? extends Serializable>> getSteps() {+ return Lists.newArrayList(serverStep, workspaceStep, repositoriesStep, optionStep);}-+}\ No newline at end of filediff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectImportOption.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportOption.javasimilarity index 94%rename from server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectImportOption.javarename to server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportOption.javaindex 44a12ce103..4cdff02305 100644--- a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectImportOption.java+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportOption.java@@ -11,7 +11,7 @@ import io.onedev.server.web.editable.annotation.Editable;import io.onedev.server.web.editable.annotation.RoleChoice;@Editable-public class ProjectImportOption implements Serializable {+public class ImportOption implements Serializable {private static final long serialVersionUID = 1L;diff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectImportSource.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportRepositories.javasimilarity index 94%rename from server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectImportSource.javarename to server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportRepositories.javaindex f58970685e..bc8c999a9e 100644--- a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectImportSource.java+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportRepositories.java@@ -15,10 +15,11 @@ import io.onedev.server.model.Project;import io.onedev.server.util.validation.Validatable;import io.onedev.server.util.validation.annotation.ClassValidating;import io.onedev.server.web.editable.annotation.Editable;+import io.onedev.server.web.editable.annotation.OmitName;@Editable@ClassValidating-public class ProjectImportSource implements Serializable, Validatable {+public class ImportRepositories implements Serializable, Validatable {private static final long serialVersionUID = 1L;@@ -26,6 +27,7 @@ public class ProjectImportSource implements Serializable, Validatable {@Editable(order=100, name="Repositories to Import")@Size(min=1, max=10000, message="No repositories to import")+ @OmitNamepublic List<ProjectMapping> getProjectMappings() {return projectMappings;}diff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportServer.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportServer.javaindex 8fd6bbf019..fe452bd0e8 100644--- a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportServer.java+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportServer.java@@ -1,6 +1,13 @@package io.onedev.server.plugin.imports.bitbucketcloud;import java.io.Serializable;+import java.net.URI;+import java.net.URISyntaxException;+import java.util.ArrayList;+import java.util.Collection;+import java.util.HashMap;+import java.util.List;+import java.util.Map;import javax.validation.ConstraintValidatorContext;import javax.ws.rs.client.Client;@@ -9,13 +16,26 @@ import javax.ws.rs.client.Invocation;import javax.ws.rs.client.WebTarget;import javax.ws.rs.core.Response;-import org.apache.wicket.MetaDataKey;+import org.apache.http.client.utils.URIBuilder;import org.glassfish.jersey.client.ClientProperties;import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;import org.hibernate.validator.constraints.NotEmpty;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;+import com.fasterxml.jackson.databind.JsonNode;+import com.google.common.base.Preconditions;++import io.onedev.commons.utils.ExplicitException;import io.onedev.commons.utils.StringUtils;+import io.onedev.commons.utils.TaskLogger;+import io.onedev.server.OneDev;+import io.onedev.server.entitymanager.ProjectManager;+import io.onedev.server.model.Project;+import io.onedev.server.storage.StorageManager;+import io.onedev.server.util.CollectionUtils;import io.onedev.server.util.JerseyUtils;+import io.onedev.server.util.JerseyUtils.PageDataConsumer;import io.onedev.server.util.validation.Validatable;import io.onedev.server.util.validation.annotation.ClassValidating;import io.onedev.server.web.editable.annotation.Editable;@@ -27,11 +47,9 @@ public class ImportServer implements Serializable, Validatable {private static final long serialVersionUID = 1L;- static final MetaDataKey<ImportServer> META_DATA_KEY = new MetaDataKey<ImportServer>() {-- private static final long serialVersionUID = 1L;-- };+ private static final Logger logger = LoggerFactory.getLogger(ImportServer.class);++ private static final int PER_PAGE = 50;private String userName;@@ -59,10 +77,159 @@ public class ImportServer implements Serializable, Validatable {this.appPassword = appPassword;}- public String getApiEndpoint(String apiPath) {+ private String getApiEndpoint(String apiPath) {return "https://api.bitbucket.org/2.0/" + StringUtils.stripStart(apiPath, "/");}++ Map<String, String> listWorkspaces() {+ Map<String, String> workspaces = new HashMap<>();++ Client client = newClient();+ try {+ String apiEndpoint = getApiEndpoint("/user/permissions/workspaces");+ for (JsonNode valueElementNode: list(client, apiEndpoint, new TaskLogger() {++ @Override+ public void log(String message, String sessionId) {+ logger.info(message);+ }++ })) {+ JsonNode workspaceNode = valueElementNode.get("workspace");+ workspaces.put(workspaceNode.get("slug").asText(), workspaceNode.get("name").asText());+ }+ CollectionUtils.sortByValue(workspaces);+ } catch (Exception e) {+ logger.error("Error listing workspaces", e);+ } finally {+ client.close();+ }++ return workspaces;+ }++ List<String> listRepositories(String workspaceId) {+ Client client = newClient();+ try {+ List<String> repositories = new ArrayList<>();++ String apiEndpoint = getApiEndpoint("/repositories/" + workspaceId);+ for (JsonNode repoNode: list(client, apiEndpoint, new TaskLogger() {++ @Override+ public void log(String message, String sessionId) {+ logger.info(message);+ }++ })) {+ repositories.add(repoNode.get("full_name").asText());+ }++ return repositories;+ } finally {+ client.close();+ }+ }++ private List<JsonNode> list(Client client, String apiEndpoint, TaskLogger logger) {+ List<JsonNode> result = new ArrayList<>();+ list(client, apiEndpoint, new PageDataConsumer() {++ @Override+ public void consume(List<JsonNode> pageData) {+ result.addAll(pageData);+ }++ }, logger);+ return result;+ }++ private void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer,+ TaskLogger logger) {+ URI uri;+ try {+ uri = new URIBuilder(apiEndpoint)+ .addParameter("pagelen", String.valueOf(PER_PAGE)).build();+ } catch (URISyntaxException e) {+ throw new RuntimeException(e);+ }++ while (true) {+ try {+ List<JsonNode> pageData = new ArrayList<>();+ JsonNode resultNode = JerseyUtils.get(client, uri.toString(), logger);+ if (resultNode.hasNonNull("values")) {+ for (JsonNode each: resultNode.get("values"))+ pageData.add(each);+ pageDataConsumer.consume(pageData);+ }+ if (resultNode.hasNonNull("next"))+ uri = new URIBuilder(resultNode.get("next").asText()).build();+ else+ break;+ } catch (URISyntaxException | InterruptedException e) {+ throw new RuntimeException(e);+ }+ }+ }++ String doImport(ImportRepositories repositories, ImportOption option, boolean dryRun, TaskLogger logger) {+ Collection<Long> projectIds = new ArrayList<>();++ Client client = newClient();+ try {+ for (ProjectMapping projectMapping: repositories.getProjectMappings()) {+ logger.log("Cloning code from repository " + projectMapping.getBitbucketRepo() + "...");++ String apiEndpoint = getApiEndpoint("/repositories/" + projectMapping.getBitbucketRepo());+ JsonNode repoNode = JerseyUtils.get(client, apiEndpoint, logger);+ ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);+ Project project = projectManager.initialize(projectMapping.getOneDevProject());+ Preconditions.checkState(project.isNew());++ project.setDescription(repoNode.get("description").asText(null));++ boolean isPrivate = repoNode.get("is_private").asBoolean();+ if (!isPrivate && option.getPublicRole() != null)+ project.setDefaultRole(option.getPublicRole());+ String cloneUrl = null;+ for (JsonNode cloneNode: repoNode.get("links").get("clone")) {+ if (cloneNode.get("name").asText().equals("https")) {+ cloneUrl = cloneNode.get("href").asText();+ break;+ }+ }+ if (cloneUrl == null)+ throw new ExplicitException("Https clone url not found");++ URIBuilder builder = new URIBuilder(cloneUrl);+ if (isPrivate)+ builder.setUserInfo(getUserName(), getAppPassword());++ if (!dryRun) {+ projectManager.clone(project, builder.build().toString());+ projectIds.add(project.getId());+ }+ }++ return "Repositories imported successfully";+ } catch (Exception e) {+ for (Long projectId: projectIds)+ OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);+ throw new RuntimeException(e);+ } finally {+ client.close();+ }+ }++ private Client newClient() {+ Client client = ClientBuilder.newClient();+ client.property(ClientProperties.FOLLOW_REDIRECTS, true);+ client.register(HttpAuthenticationFeature.basic(getUserName(), getAppPassword()));+ return client;+ }+@Overridepublic boolean isValid(ConstraintValidatorContext context) {Client client = ClientBuilder.newClient();@@ -99,11 +266,4 @@ public class ImportServer implements Serializable, Validatable {return true;}- Client newClient() {- Client client = ClientBuilder.newClient();- client.property(ClientProperties.FOLLOW_REDIRECTS, true);- client.register(HttpAuthenticationFeature.basic(getUserName(), getAppPassword()));- return client;- }-}diff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportUtils.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportUtils.javadeleted file mode 100644index 0810d5cecc..0000000000--- a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportUtils.java+++ /dev/null@@ -1,66 +0,0 @@-package io.onedev.server.plugin.imports.bitbucketcloud;--import java.net.URI;-import java.net.URISyntaxException;-import java.util.ArrayList;-import java.util.List;--import javax.ws.rs.client.Client;--import org.apache.http.client.utils.URIBuilder;--import com.fasterxml.jackson.databind.JsonNode;--import io.onedev.commons.utils.TaskLogger;-import io.onedev.server.util.JerseyUtils;-import io.onedev.server.util.JerseyUtils.PageDataConsumer;--public class ImportUtils {-- static final String NAME = "Bitbucket Cloud";-- static final int PER_PAGE = 50;-- static List<JsonNode> list(Client client, String apiEndpoint, TaskLogger logger) {- List<JsonNode> result = new ArrayList<>();- list(client, apiEndpoint, new PageDataConsumer() {-- @Override- public void consume(List<JsonNode> pageData) {- result.addAll(pageData);- }-- }, logger);- return result;- }-- static void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer,- TaskLogger logger) {- URI uri;- try {- uri = new URIBuilder(apiEndpoint)- .addParameter("pagelen", String.valueOf(PER_PAGE)).build();- } catch (URISyntaxException e) {- throw new RuntimeException(e);- }-- while (true) {- try {- List<JsonNode> pageData = new ArrayList<>();- JsonNode resultNode = JerseyUtils.get(client, uri.toString(), logger);- if (resultNode.hasNonNull("values")) {- for (JsonNode each: resultNode.get("values"))- pageData.add(each);- pageDataConsumer.consume(pageData);- }- if (resultNode.hasNonNull("next"))- uri = new URIBuilder(resultNode.get("next").asText()).build();- else- break;- } catch (URISyntaxException | InterruptedException e) {- throw new RuntimeException(e);- }- }- }--}\ No newline at end of filediff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportWorkspace.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportWorkspace.javanew file mode 100644index 0000000000..d841d12dd7--- /dev/null+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportWorkspace.java@@ -0,0 +1,40 @@+package io.onedev.server.plugin.imports.bitbucketcloud;++import java.io.Serializable;+import java.util.Map;++import org.hibernate.validator.constraints.NotEmpty;++import io.onedev.server.util.ComponentContext;+import io.onedev.server.web.editable.BeanEditor;+import io.onedev.server.web.editable.annotation.ChoiceProvider;+import io.onedev.server.web.editable.annotation.Editable;++@Editable+public class ImportWorkspace implements Serializable {++ private static final long serialVersionUID = 1L;++ ImportServer server;++ private String workspace;++ @Editable(order=100, name="Bitbucket Workspace", description="Select workspace to import from")+ @ChoiceProvider("getWorkspaceChoices")+ @NotEmpty+ public String getWorkspace() {+ return workspace;+ }++ public void setWorkspace(String workspace) {+ this.workspace = workspace;+ }++ @SuppressWarnings("unused")+ private static Map<String, String> getWorkspaceChoices() {+ BeanEditor editor = ComponentContext.get().getComponent().findParent(BeanEditor.class);+ ImportWorkspace setting = (ImportWorkspace) editor.getModelObject();+ return setting.server.listWorkspaces();+ }++}diff --git a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaIssueImporter.java b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaIssueImporter.javaindex 3f15c05387..69a63bc61c 100644--- a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaIssueImporter.java+++ b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaIssueImporter.java@@ -1,6 +1,8 @@package io.onedev.server.plugin.imports.gitea;+import java.io.Serializable;import java.util.HashMap;+import java.util.List;import java.util.Map;import java.util.Optional;@@ -10,35 +12,84 @@ import io.onedev.commons.utils.TaskLogger;import io.onedev.server.imports.IssueImporter;import io.onedev.server.model.Project;import io.onedev.server.model.User;-import io.onedev.server.web.util.WicketUtils;+import io.onedev.server.web.util.ImportStep;-public class GiteaIssueImporter extends IssueImporter<ImportServer, IssueImportSource, IssueImportOption> {+public class GiteaIssueImporter implements IssueImporter {private static final long serialVersionUID = 1L;- @Override- public String getName() {- return ImportUtils.NAME;- }+ private final ImportStep<ImportServer> serverStep = new ImportStep<ImportServer>() {- @Override- public IssueImportSource getWhat(ImportServer where, TaskLogger logger) {- WicketUtils.getPage().setMetaData(ImportServer.META_DATA_KEY, where);- return new IssueImportSource();- }+ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Authenticate to Gitea";+ }++ @Override+ protected ImportServer newSetting() {+ return new ImportServer();+ }++ };++ private final ImportStep<ImportRepository> repositoryStep = new ImportStep<ImportRepository>() {++ private static final long serialVersionUID = 1L;+ @Override+ public String getTitle() {+ return "Choose repository";+ }++ @Override+ protected ImportRepository newSetting() {+ ImportRepository repository = new ImportRepository();+ repository.server = serverStep.getSetting();+ return repository;+ }++ };++ private final ImportStep<IssueImportOption> optionStep = new ImportStep<IssueImportOption>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Specify import option";+ }++ @Override+ protected IssueImportOption newSetting() {+ return serverStep.getSetting().buildIssueImportOption(+ Lists.newArrayList(repositoryStep.getSetting().getRepository()));+ }++ };+@Override- public IssueImportOption getHow(ImportServer where, IssueImportSource what, TaskLogger logger) {- return ImportUtils.buildIssueImportOption(where, Lists.newArrayList(what.getRepository()), logger);+ public String getName() {+ return GiteaPluginModule.NAME;}@Override- public String doImport(ImportServer where, IssueImportSource what, IssueImportOption how,- Project project, boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {- logger.log("Importing issues from repository " + what.getRepository() + "...");+ public String doImport(Project project, boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {+ ImportServer server = serverStep.getSetting();+ String giteaRepo = repositoryStep.getSetting().getRepository();+ IssueImportOption option = optionStep.getSetting();++ logger.log("Importing issues from repository " + giteaRepo + "...");Map<String, Optional<User>> users = new HashMap<>();- return ImportUtils.importIssues(where, what.getRepository(), project, retainIssueNumbers, how, users, dryRun, logger)++ return server.importIssues(giteaRepo, project, retainIssueNumbers, option, users, dryRun, logger).toHtml("Issues imported successfully");}+ @Override+ public List<ImportStep<? extends Serializable>> getSteps() {+ return Lists.newArrayList(serverStep, repositoryStep, optionStep);+ }+}diff --git a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaPluginModule.java b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaPluginModule.javaindex 4e87a4224a..f060bdfb82 100644--- a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaPluginModule.java+++ b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaPluginModule.java@@ -1,6 +1,5 @@package io.onedev.server.plugin.imports.gitea;-import java.io.Serializable;import java.util.Collection;import com.google.common.collect.Lists;@@ -17,6 +16,8 @@ import io.onedev.server.imports.ProjectImporterContribution;*/public class GiteaPluginModule extends AbstractPluginModule {+ static final String NAME = "Gitea";+@Overrideprotected void configure() {super.configure();@@ -25,7 +26,7 @@ public class GiteaPluginModule extends AbstractPluginModule {contribute(ProjectImporterContribution.class, new ProjectImporterContribution() {@Override- public Collection<ProjectImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> getImporters() {+ public Collection<ProjectImporter> getImporters() {return Lists.newArrayList(new GiteaProjectImporter());}@@ -39,7 +40,7 @@ public class GiteaPluginModule extends AbstractPluginModule {contribute(IssueImporterContribution.class, new IssueImporterContribution() {@Override- public Collection<IssueImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> getImporters() {+ public Collection<IssueImporter> getImporters() {return Lists.newArrayList(new GiteaIssueImporter());}diff --git a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaProjectImporter.java b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaProjectImporter.javaindex 3646bce53e..42aa2f616d 100644--- a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaProjectImporter.java+++ b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaProjectImporter.java@@ -1,141 +1,112 @@package io.onedev.server.plugin.imports.gitea;-import java.util.ArrayList;-import java.util.Collection;-import java.util.HashMap;+import java.io.Serializable;import java.util.List;-import java.util.Map;-import java.util.Optional;import java.util.stream.Collectors;-import javax.ws.rs.client.Client;--import org.apache.http.client.utils.URIBuilder;-import org.joda.time.format.ISODateTimeFormat;--import com.fasterxml.jackson.databind.JsonNode;-import com.google.common.base.Preconditions;+import com.google.common.collect.Lists;import io.onedev.commons.utils.TaskLogger;-import io.onedev.server.OneDev;-import io.onedev.server.entitymanager.MilestoneManager;-import io.onedev.server.entitymanager.ProjectManager;import io.onedev.server.imports.ProjectImporter;-import io.onedev.server.model.Milestone;-import io.onedev.server.model.Project;-import io.onedev.server.model.User;-import io.onedev.server.storage.StorageManager;-import io.onedev.server.util.JerseyUtils;+import io.onedev.server.web.util.ImportStep;-public class GiteaProjectImporter extends ProjectImporter<ImportServer, ProjectImportSource, ProjectImportOption> {+public class GiteaProjectImporter implements ProjectImporter {private static final long serialVersionUID = 1L;++ private final ImportStep<ImportServer> serverStep = new ImportStep<ImportServer>() {- @Override- public String getName() {- return ImportUtils.NAME;- }+ private static final long serialVersionUID = 1L;- @Override- public ProjectImportSource getWhat(ImportServer where, TaskLogger logger) {- ProjectImportSource importSource = new ProjectImportSource();- Client client = where.newClient();- try {- String apiEndpoint = where.getApiEndpoint("/user/repos");- for (JsonNode repoNode: ImportUtils.list(client, apiEndpoint, logger)) {- String repoName = repoNode.get("name").asText();- String ownerName = repoNode.get("owner").get("login").asText();+ @Override+ public String getTitle() {+ return "Authenticate to Gitea";+ }++ @Override+ protected ImportServer newSetting() {+ return new ImportServer();+ }++ };++ private final ImportStep<ImportOrganization> organizationStep = new ImportStep<ImportOrganization>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Choose organization";+ }++ @Override+ protected ImportOrganization newSetting() {+ ImportOrganization organization = new ImportOrganization();+ organization.server = serverStep.getSetting();+ return organization;+ }++ };++ private final ImportStep<ImportRepositories> repositoriesStep = new ImportStep<ImportRepositories>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Specify repositories";+ }++ @Override+ protected ImportRepositories newSetting() {+ ImportRepositories repositories = new ImportRepositories();+ String organization = organizationStep.getSetting().getOrganization();+ for (String repository: serverStep.getSetting().listRepositories(organization)) {ProjectMapping projectMapping = new ProjectMapping();- projectMapping.setGiteaRepo(ownerName + "/" + repoName);- projectMapping.setOneDevProject(ownerName + "/" + repoName);- importSource.getProjectMappings().add(projectMapping);- }- } finally {- client.close();+ projectMapping.setGiteaRepo(repository);+ projectMapping.setOneDevProject(repository);+ repositories.getProjectMappings().add(projectMapping);+ }+ return repositories;}- return importSource;++ };++ private final ImportStep<ProjectImportOption> optionStep = new ImportStep<ProjectImportOption>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Specify import option";+ }++ @Override+ protected ProjectImportOption newSetting() {+ ProjectImportOption option = new ProjectImportOption();+ List<String> giteaRepos = repositoriesStep.getSetting().getProjectMappings().stream()+ .map(it->it.getGiteaRepo()).collect(Collectors.toList());+ option.setIssueImportOption(serverStep.getSetting().buildIssueImportOption(giteaRepos));+ return option;+ }++ };++ @Override+ public String getName() {+ return GiteaPluginModule.NAME;}@Override- public ProjectImportOption getHow(ImportServer where, ProjectImportSource what, TaskLogger logger) {- ProjectImportOption importOption = new ProjectImportOption();- List<String> repos = what.getProjectMappings().stream().map(it->it.getGiteaRepo()).collect(Collectors.toList());- importOption.setIssueImportOption(ImportUtils.buildIssueImportOption(where, repos, logger));- return importOption;+ public String doImport(boolean dryRun, TaskLogger logger) {+ ImportRepositories repositories = repositoriesStep.getSetting();+ ProjectImportOption option = optionStep.getSetting();+ return serverStep.getSetting().importProjects(repositories, option, dryRun, logger);}@Override- public String doImport(ImportServer where, ProjectImportSource what, ProjectImportOption how,- boolean dryRun, TaskLogger logger) {- Collection<Long> projectIds = new ArrayList<>();- Client client = where.newClient();- try {- Map<String, Optional<User>> users = new HashMap<>();- ImportResult result = new ImportResult();- for (ProjectMapping projectMapping: what.getProjectMappings()) {- logger.log("Cloning code from repository " + projectMapping.getGiteaRepo() + "...");-- String apiEndpoint = where.getApiEndpoint("/repos/" + projectMapping.getGiteaRepo());- JsonNode repoNode = JerseyUtils.get(client, apiEndpoint, logger);- ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);- Project project = projectManager.initialize(projectMapping.getOneDevProject());- Preconditions.checkState(project.isNew());- project.setDescription(repoNode.get("description").asText(null));- project.setIssueManagement(repoNode.get("has_issues").asBoolean());-- boolean isPrivate = repoNode.get("private").asBoolean();- if (!isPrivate && how.getPublicRole() != null)- project.setDefaultRole(how.getPublicRole());-- URIBuilder builder = new URIBuilder(repoNode.get("clone_url").asText());- if (isPrivate)- builder.setUserInfo("git", where.getAccessToken());-- if (!dryRun) {- projectManager.clone(project, builder.build().toString());- projectIds.add(project.getId());- }-- if (how.getIssueImportOption() != null) {- List<Milestone> milestones = new ArrayList<>();- logger.log("Importing milestones from repository " + projectMapping.getGiteaRepo() + "...");- apiEndpoint = where.getApiEndpoint("/repos/" + projectMapping.getGiteaRepo() + "/milestones?state=all");- for (JsonNode milestoneNode: ImportUtils.list(client, apiEndpoint, logger)) {- Milestone milestone = new Milestone();- milestone.setName(milestoneNode.get("title").asText());- milestone.setDescription(milestoneNode.get("description").asText(null));- milestone.setProject(project);- String dueDateString = milestoneNode.get("due_on").asText(null);- if (dueDateString != null)- milestone.setDueDate(ISODateTimeFormat.dateTimeNoMillis().parseDateTime(dueDateString).toDate());- if (milestoneNode.get("state").asText().equals("closed"))- milestone.setClosed(true);-- milestones.add(milestone);- project.getMilestones().add(milestone);-- if (!dryRun)- OneDev.getInstance(MilestoneManager.class).save(milestone);- }-- logger.log("Importing issues from repository " + projectMapping.getGiteaRepo() + "...");- ImportResult currentResult = ImportUtils.importIssues(where, projectMapping.getGiteaRepo(),- project, true, how.getIssueImportOption(), users, dryRun, logger);- result.nonExistentLogins.addAll(currentResult.nonExistentLogins);- result.nonExistentMilestones.addAll(currentResult.nonExistentMilestones);- result.unmappedIssueLabels.addAll(currentResult.unmappedIssueLabels);- result.issuesImported |= currentResult.issuesImported;- }- }-- return result.toHtml("Repositories imported successfully");- } catch (Exception e) {- for (Long projectId: projectIds)- OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);- throw new RuntimeException(e);- } finally {- client.close();- }+ public List<ImportStep<? extends Serializable>> getSteps() {+ return Lists.newArrayList(serverStep, organizationStep, repositoriesStep, optionStep);}}diff --git a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportOrganization.java b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportOrganization.javanew file mode 100644index 0000000000..3f754db584--- /dev/null+++ b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportOrganization.java@@ -0,0 +1,38 @@+package io.onedev.server.plugin.imports.gitea;++import java.io.Serializable;+import java.util.List;++import io.onedev.server.util.ComponentContext;+import io.onedev.server.web.editable.BeanEditor;+import io.onedev.server.web.editable.annotation.ChoiceProvider;+import io.onedev.server.web.editable.annotation.Editable;++@Editable+public class ImportOrganization implements Serializable {++ private static final long serialVersionUID = 1L;++ ImportServer server;++ private String organization;++ @Editable(order=100, name="Gitea Organization", description="Select organization to import from. "+ + "Leave empty to import from repositories under current account")+ @ChoiceProvider("getOrganizationChoices")+ public String getOrganization() {+ return organization;+ }++ public void setOrganization(String organization) {+ this.organization = organization;+ }++ @SuppressWarnings("unused")+ private static List<String> getOrganizationChoices() {+ BeanEditor editor = ComponentContext.get().getComponent().findParent(BeanEditor.class);+ ImportOrganization setting = (ImportOrganization) editor.getModelObject();+ return setting.server.listOrganizations();+ }++}diff --git a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ProjectImportSource.java b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportRepositories.javasimilarity index 94%rename from server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ProjectImportSource.javarename to server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportRepositories.javaindex 0ceea04233..f980982743 100644--- a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ProjectImportSource.java+++ b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportRepositories.java@@ -15,10 +15,11 @@ import io.onedev.server.model.Project;import io.onedev.server.util.validation.Validatable;import io.onedev.server.util.validation.annotation.ClassValidating;import io.onedev.server.web.editable.annotation.Editable;+import io.onedev.server.web.editable.annotation.OmitName;@Editable@ClassValidating-public class ProjectImportSource implements Serializable, Validatable {+public class ImportRepositories implements Serializable, Validatable {private static final long serialVersionUID = 1L;@@ -26,6 +27,7 @@ public class ProjectImportSource implements Serializable, Validatable {@Editable(order=100, name="Repositories to Import")@Size(min=1, max=10000, message="No repositories to import")+ @OmitNamepublic List<ProjectMapping> getProjectMappings() {return projectMappings;}diff --git a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportRepository.java b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportRepository.javanew file mode 100644index 0000000000..2e7d46ae71--- /dev/null+++ b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportRepository.java@@ -0,0 +1,39 @@+package io.onedev.server.plugin.imports.gitea;++import java.util.List;++import org.hibernate.validator.constraints.NotEmpty;++import io.onedev.server.util.ComponentContext;+import io.onedev.server.util.EditContext;+import io.onedev.server.web.editable.BeanEditor;+import io.onedev.server.web.editable.annotation.ChoiceProvider;+import io.onedev.server.web.editable.annotation.Editable;++@Editable+public class ImportRepository extends ImportOrganization {++ private static final long serialVersionUID = 1L;++ private String repository;++ @Editable(order=200, name="Gitea Repository", description="Select repository to import from")+ @ChoiceProvider("getRepositoryChoices")+ @NotEmpty+ public String getRepository() {+ return repository;+ }++ public void setRepository(String repository) {+ this.repository = repository;+ }++ @SuppressWarnings("unused")+ private static List<String> getRepositoryChoices() {+ BeanEditor editor = ComponentContext.get().getComponent().findParent(BeanEditor.class);+ ImportRepository setting = (ImportRepository) editor.getModelObject();+ String organization = (String) EditContext.get().getInputValue("organization");+ return setting.server.listRepositories(organization);+ }++}diff --git a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportServer.java b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportServer.javaindex 29b5266fcd..86ca901faa 100644--- a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportServer.java+++ b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportServer.java@@ -1,7 +1,24 @@package io.onedev.server.plugin.imports.gitea;import java.io.Serializable;+import java.net.URI;+import java.net.URISyntaxException;+import java.time.Instant;+import java.time.format.DateTimeFormatter;+import java.util.ArrayList;+import java.util.Collection;+import java.util.Date;+import java.util.HashMap;+import java.util.HashSet;+import java.util.LinkedHashMap;+import java.util.LinkedHashSet;+import java.util.List;+import java.util.Map;+import java.util.Optional;+import java.util.Set;+import java.util.concurrent.atomic.AtomicInteger;+import javax.annotation.Nullable;import javax.validation.ConstraintValidatorContext;import javax.ws.rs.client.Client;import javax.ws.rs.client.ClientBuilder;@@ -9,13 +26,46 @@ import javax.ws.rs.client.Invocation;import javax.ws.rs.client.WebTarget;import javax.ws.rs.core.Response;-import org.apache.wicket.MetaDataKey;+import org.apache.http.client.utils.URIBuilder;import org.glassfish.jersey.client.ClientProperties;import org.glassfish.jersey.client.oauth2.OAuth2ClientSupport;import org.hibernate.validator.constraints.NotEmpty;+import org.joda.time.format.ISODateTimeFormat;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;+import org.unbescape.html.HtmlEscape;+import com.fasterxml.jackson.databind.JsonNode;+import com.google.common.base.Preconditions;++import edu.emory.mathcs.backport.java.util.Collections;+import io.onedev.commons.utils.ExplicitException;import io.onedev.commons.utils.StringUtils;+import io.onedev.commons.utils.TaskLogger;+import io.onedev.server.OneDev;+import io.onedev.server.entitymanager.IssueManager;+import io.onedev.server.entitymanager.MilestoneManager;+import io.onedev.server.entitymanager.ProjectManager;+import io.onedev.server.entitymanager.SettingManager;+import io.onedev.server.entitymanager.UserManager;+import io.onedev.server.entityreference.ReferenceMigrator;+import io.onedev.server.model.Issue;+import io.onedev.server.model.IssueComment;+import io.onedev.server.model.IssueField;+import io.onedev.server.model.IssueSchedule;+import io.onedev.server.model.Milestone;+import io.onedev.server.model.Project;+import io.onedev.server.model.User;+import io.onedev.server.model.support.LastUpdate;+import io.onedev.server.model.support.administration.GlobalIssueSetting;+import io.onedev.server.model.support.inputspec.InputSpec;+import io.onedev.server.model.support.issue.field.spec.FieldSpec;+import io.onedev.server.persistence.dao.Dao;+import io.onedev.server.storage.StorageManager;+import io.onedev.server.util.DateUtils;import io.onedev.server.util.JerseyUtils;+import io.onedev.server.util.JerseyUtils.PageDataConsumer;+import io.onedev.server.util.Pair;import io.onedev.server.util.validation.Validatable;import io.onedev.server.util.validation.annotation.ClassValidating;import io.onedev.server.web.editable.annotation.Editable;@@ -27,15 +77,13 @@ public class ImportServer implements Serializable, Validatable {private static final long serialVersionUID = 1L;- static final MetaDataKey<ImportServer> META_DATA_KEY = new MetaDataKey<ImportServer>() {-- private static final long serialVersionUID = 1L;-- };-- static final String PROP_API_URL = "apiUrl";+ private static final Logger logger = LoggerFactory.getLogger(ImportServer.class);- static final String PROP_ACCESS_TOKEN = "accessToken";+ private static final int PER_PAGE = 50;++ private static final String PROP_API_URL = "apiUrl";++ private static final String PROP_ACCESS_TOKEN = "accessToken";private String apiUrl;@@ -62,10 +110,502 @@ public class ImportServer implements Serializable, Validatable {this.accessToken = accessToken;}- public String getApiEndpoint(String apiPath) {+ private String getApiEndpoint(String apiPath) {return StringUtils.stripEnd(apiUrl, "/") + "/" + StringUtils.stripStart(apiPath, "/");}+ List<String> listOrganizations() {+ List<String> organizations = new ArrayList<>();++ Client client = newClient();+ try {+ String apiEndpoint = getApiEndpoint("/user/orgs");+ for (JsonNode orgNode: list(client, apiEndpoint, new TaskLogger() {++ @Override+ public void log(String message, String sessionId) {+ logger.info(message);+ }++ })) {+ organizations.add(orgNode.get("username").asText());+ }+ Collections.sort(organizations);+ } catch (Exception e) {+ logger.error("Error listing organizations", e);+ } finally {+ client.close();+ }++ return organizations;+ }++ List<String> listRepositories(@Nullable String organization) {+ Client client = newClient();+ try {+ String apiEndpoint;+ if (organization != null)+ apiEndpoint = getApiEndpoint("/orgs/" + organization + "/repos");+ else+ apiEndpoint = getApiEndpoint("/user/repos");+ List<String> repositories = new ArrayList<>();+ for (JsonNode repoNode: list(client, apiEndpoint, new TaskLogger() {++ @Override+ public void log(String message, String sessionId) {+ logger.info(message);+ }++ })) {+ String repoName = repoNode.get("name").asText();+ JsonNode ownerNode = repoNode.get("owner");+ String ownerName = ownerNode.get("login").asText();+ JsonNode emailNode = ownerNode.get("email");+ if (organization != null || emailNode != null && StringUtils.isNotBlank(emailNode.asText()))+ repositories.add(ownerName + "/" + repoName);+ }+ Collections.sort(repositories);+ return repositories;+ } finally {+ client.close();+ }++ }++ IssueImportOption buildIssueImportOption(Collection<String> repositories) {+ IssueImportOption option = new IssueImportOption();+ Client client = newClient();+ try {+ TaskLogger taskLogger = new TaskLogger() {++ @Override+ public void log(String message, String sessionId) {+ logger.info(message);+ }++ };+ Set<String> labels = new LinkedHashSet<>();+ for (String repo: repositories) {+ String apiEndpoint = getApiEndpoint("/repos/" + repo + "/labels");+ for (JsonNode labelNode: list(client, apiEndpoint, taskLogger))+ labels.add(labelNode.get("name").asText());+ try {+ apiEndpoint = getApiEndpoint("/orgs/" + StringUtils.substringBefore(repo, "/") + "/labels");+ for (JsonNode labelNode: list(client, apiEndpoint, taskLogger))+ labels.add(labelNode.get("name").asText());+ } catch (Exception e) {+ // ignore as exception might be thrown if repo belongs to a user account+ }+ }++ for (String label: labels) {+ IssueLabelMapping mapping = new IssueLabelMapping();+ mapping.setGiteaIssueLabel(label);+ option.getIssueLabelMappings().add(mapping);+ }+ } finally {+ client.close();+ }+ return option;+ }++ @Nullable+ private User getUser(Map<String, Optional<User>> users, JsonNode userNode, TaskLogger logger) {+ String login = userNode.get("login").asText();+ Optional<User> userOpt = users.get(login);+ if (userOpt == null) {+ String email = null;+ if (userNode.hasNonNull("email"))+ email = userNode.get("email").asText(null);+ if (email != null)+ userOpt = Optional.ofNullable(OneDev.getInstance(UserManager.class).findByVerifiedEmailAddress(email));+ else+ userOpt = Optional.empty();+ users.put(login, userOpt);+ }+ return userOpt.orElse(null);+ }++ ImportResult importIssues(String giteaRepo, Project oneDevProject, boolean useExistingIssueNumbers,+ IssueImportOption option, Map<String, Optional<User>> users, boolean dryRun, TaskLogger logger) {+ Client client = newClient();+ try {+ Set<String> nonExistentMilestones = new HashSet<>();+ Set<String> nonExistentLogins = new HashSet<>();+ Set<String> unmappedIssueLabels = new HashSet<>();++ Map<String, Pair<FieldSpec, String>> labelMappings = new HashMap<>();+ Map<String, Milestone> milestoneMappings = new HashMap<>();++ for (IssueLabelMapping mapping: option.getIssueLabelMappings()) {+ String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");+ String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");+ FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);+ if (fieldSpec == null)+ throw new ExplicitException("No field spec found: " + oneDevFieldName);+ labelMappings.put(mapping.getGiteaIssueLabel(), new Pair<>(fieldSpec, oneDevFieldValue));+ }++ for (Milestone milestone: oneDevProject.getMilestones())+ milestoneMappings.put(milestone.getName(), milestone);++ String initialIssueState = getIssueSetting().getInitialStateSpec().getName();++ List<Issue> issues = new ArrayList<>();++ Map<Long, Long> issueNumberMappings = new HashMap<>();++ AtomicInteger numOfImportedIssues = new AtomicInteger(0);+ PageDataConsumer pageDataConsumer = new PageDataConsumer() {++ private String joinAsMultilineHtml(List<String> values) {+ List<String> escapedValues = new ArrayList<>();+ for (String value: values)+ escapedValues.add(HtmlEscape.escapeHtml5(value));+ return StringUtils.join(escapedValues, "<br>");+ }++ @Override+ public void consume(List<JsonNode> pageData) throws InterruptedException {+ for (JsonNode issueNode: pageData) {+ if (Thread.interrupted())+ throw new InterruptedException();++ Map<String, String> extraIssueInfo = new LinkedHashMap<>();++ Issue issue = new Issue();+ issue.setProject(oneDevProject);+ issue.setTitle(issueNode.get("title").asText());+ issue.setDescription(issueNode.get("body").asText(null));++ issue.setNumberScope(oneDevProject.getForkRoot());++ Long oldNumber = issueNode.get("number").asLong();+ Long newNumber;+ if (dryRun || useExistingIssueNumbers)+ newNumber = oldNumber;+ else+ newNumber = OneDev.getInstance(IssueManager.class).getNextNumber(oneDevProject);+ issue.setNumber(newNumber);+ issueNumberMappings.put(oldNumber, newNumber);++ String issueFQN = giteaRepo + "#" + oldNumber;++ if (issueNode.get("state").asText().equals("closed"))+ issue.setState(option.getClosedIssueState());+ else+ issue.setState(initialIssueState);++ if (issueNode.hasNonNull("milestone")) {+ String milestoneName = issueNode.get("milestone").get("title").asText();+ Milestone milestone = milestoneMappings.get(milestoneName);+ if (milestone != null) {+ IssueSchedule schedule = new IssueSchedule();+ schedule.setIssue(issue);+ schedule.setMilestone(milestone);+ issue.getSchedules().add(schedule);+ } else {+ extraIssueInfo.put("Milestone", milestoneName);+ nonExistentMilestones.add(milestoneName);+ }+ }++ JsonNode userNode = issueNode.get("user");+ User user = getUser(users, userNode, logger);+ if (user != null) {+ issue.setSubmitter(user);+ } else {+ issue.setSubmitter(OneDev.getInstance(UserManager.class).getUnknown());+ nonExistentLogins.add(userNode.get("login").asText());+ }++ String created_at = issueNode.get("created_at").asText();+ issue.setSubmitDate(Date.from(Instant.from(+ DateTimeFormatter.ISO_DATE_TIME.parse(created_at))));++ LastUpdate lastUpdate = new LastUpdate();+ lastUpdate.setActivity("Opened");+ lastUpdate.setDate(issue.getSubmitDate());+ lastUpdate.setUser(issue.getSubmitter());+ issue.setLastUpdate(lastUpdate);++ List<JsonNode> assigneeNodes = new ArrayList<>();+ if (issueNode.hasNonNull("assignees")) {+ for (JsonNode assigneeNode: issueNode.get("assignees"))+ assigneeNodes.add(assigneeNode);+ } else if (issueNode.hasNonNull("assignee")) {+ assigneeNodes.add(issueNode.get("assignee"));+ }++ for (JsonNode assigneeNode: assigneeNodes) {+ IssueField assigneeField = new IssueField();+ assigneeField.setIssue(issue);+ assigneeField.setName(option.getAssigneesIssueField());+ assigneeField.setType(InputSpec.USER);++ user = getUser(users, assigneeNode, logger);+ if (user != null) {+ assigneeField.setValue(user.getName());+ issue.getFields().add(assigneeField);+ } else {+ nonExistentLogins.add(assigneeNode.get("login").asText());+ }+ }++ if (option.getDueDateIssueField() != null) {+ String dueDate = issueNode.get("due_date").asText(null);+ if (dueDate != null) {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(option.getDueDateIssueField());+ issueField.setType(InputSpec.DATE);+ issueField.setValue(DateUtils.formatDate(Date.from(+ Instant.from(DateTimeFormatter.ISO_DATE_TIME.parse(dueDate)))));+ issue.getFields().add(issueField);+ }+ }++ List<String> currentUnmappedLabels = new ArrayList<>();+ for (JsonNode labelNode: issueNode.get("labels")) {+ String labelName = labelNode.get("name").asText();+ Pair<FieldSpec, String> mapped = labelMappings.get(labelName);+ if (mapped != null) {+ IssueField labelField = new IssueField();+ labelField.setIssue(issue);+ labelField.setName(mapped.getFirst().getName());+ labelField.setType(InputSpec.ENUMERATION);+ labelField.setValue(mapped.getSecond());+ labelField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));+ issue.getFields().add(labelField);+ } else {+ currentUnmappedLabels.add(labelName);+ unmappedIssueLabels.add(HtmlEscape.escapeHtml5(labelName));+ }+ }++ if (!currentUnmappedLabels.isEmpty())+ extraIssueInfo.put("Labels", joinAsMultilineHtml(currentUnmappedLabels));++ String apiEndpoint = getApiEndpoint("/repos/" + giteaRepo+ + "/issues/" + oldNumber + "/comments");+ for (JsonNode commentNode: list(client, apiEndpoint, logger)) {+ String commentContent = commentNode.get("body").asText(null);+ if (commentContent != null) {+ IssueComment comment = new IssueComment();+ comment.setIssue(issue);+ comment.setContent(commentContent);++ created_at = commentNode.get("created_at").asText();+ comment.setDate(Date.from(Instant.from(+ DateTimeFormatter.ISO_DATE_TIME.parse(created_at))));++ userNode = commentNode.get("user");+ user = getUser(users, userNode, logger);+ if (user != null) {+ comment.setUser(user);+ } else {+ comment.setUser(OneDev.getInstance(UserManager.class).getUnknown());+ nonExistentLogins.add(userNode.get("username").asText());+ }+ issue.getComments().add(comment);+ }+ }++ issue.setCommentCount(issue.getComments().size());++ Set<String> fieldAndValues = new HashSet<>();+ for (IssueField field: issue.getFields()) {+ String fieldAndValue = field.getName() + "::" + field.getValue();+ if (!fieldAndValues.add(fieldAndValue)) {+ String errorMessage = String.format(+ "Duplicate issue field mapping (issue: %s, field: %s)",+ issueFQN, fieldAndValue);+ throw new ExplicitException(errorMessage);+ }+ }++ if (!extraIssueInfo.isEmpty()) {+ StringBuilder builder = new StringBuilder("|");+ for (String key: extraIssueInfo.keySet())+ builder.append(key).append("|");+ builder.append("\n|");+ extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));+ builder.append("\n|");+ for (String value: extraIssueInfo.values())+ builder.append(value).append("|");++ if (issue.getDescription() != null)+ issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());+ else+ issue.setDescription(builder.toString());+ }+ issues.add(issue);+ }+ logger.log("Imported " + numOfImportedIssues.addAndGet(pageData.size()) + " issues");+ }++ };++ String apiEndpoint = getApiEndpoint("/repos/" + giteaRepo + "/issues?state=all&type=issues");+ list(client, apiEndpoint, pageDataConsumer, logger);++ if (!dryRun) {+ ReferenceMigrator migrator = new ReferenceMigrator(Issue.class, issueNumberMappings);+ Dao dao = OneDev.getInstance(Dao.class);+ for (Issue issue: issues) {+ if (issue.getDescription() != null)+ issue.setDescription(migrator.migratePrefixed(issue.getDescription(), "#"));++ OneDev.getInstance(IssueManager.class).save(issue);+ for (IssueSchedule schedule: issue.getSchedules())+ dao.persist(schedule);+ for (IssueField field: issue.getFields())+ dao.persist(field);+ for (IssueComment comment: issue.getComments()) {+ comment.setContent(migrator.migratePrefixed(comment.getContent(), "#"));+ dao.persist(comment);+ }+ }+ }++ ImportResult result = new ImportResult();+ result.nonExistentLogins.addAll(nonExistentLogins);+ result.nonExistentMilestones.addAll(nonExistentMilestones);+ result.unmappedIssueLabels.addAll(unmappedIssueLabels);++ if (numOfImportedIssues.get() != 0)+ result.issuesImported = true;++ return result;+ } finally {+ client.close();+ }+ }++ String importProjects(ImportRepositories repositories, ProjectImportOption option, boolean dryRun, TaskLogger logger) {+ Collection<Long> projectIds = new ArrayList<>();+ Client client = newClient();+ try {+ Map<String, Optional<User>> users = new HashMap<>();+ ImportResult result = new ImportResult();+ for (ProjectMapping projectMapping: repositories.getProjectMappings()) {+ logger.log("Cloning code from repository " + projectMapping.getGiteaRepo() + "...");++ String apiEndpoint = getApiEndpoint("/repos/" + projectMapping.getGiteaRepo());+ JsonNode repoNode = JerseyUtils.get(client, apiEndpoint, logger);+ ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);+ Project project = projectManager.initialize(projectMapping.getOneDevProject());+ Preconditions.checkState(project.isNew());+ project.setDescription(repoNode.get("description").asText(null));+ project.setIssueManagement(repoNode.get("has_issues").asBoolean());++ boolean isPrivate = repoNode.get("private").asBoolean();+ if (!isPrivate && option.getPublicRole() != null)+ project.setDefaultRole(option.getPublicRole());++ URIBuilder builder = new URIBuilder(repoNode.get("clone_url").asText());+ if (isPrivate)+ builder.setUserInfo("git", getAccessToken());++ if (!dryRun) {+ projectManager.clone(project, builder.build().toString());+ projectIds.add(project.getId());+ }++ if (option.getIssueImportOption() != null) {+ List<Milestone> milestones = new ArrayList<>();+ logger.log("Importing milestones from repository " + projectMapping.getGiteaRepo() + "...");+ apiEndpoint = getApiEndpoint("/repos/" + projectMapping.getGiteaRepo() + "/milestones?state=all");+ for (JsonNode milestoneNode: list(client, apiEndpoint, logger)) {+ Milestone milestone = new Milestone();+ milestone.setName(milestoneNode.get("title").asText());+ milestone.setDescription(milestoneNode.get("description").asText(null));+ milestone.setProject(project);+ String dueDateString = milestoneNode.get("due_on").asText(null);+ if (dueDateString != null)+ milestone.setDueDate(ISODateTimeFormat.dateTimeNoMillis().parseDateTime(dueDateString).toDate());+ if (milestoneNode.get("state").asText().equals("closed"))+ milestone.setClosed(true);++ milestones.add(milestone);+ project.getMilestones().add(milestone);++ if (!dryRun)+ OneDev.getInstance(MilestoneManager.class).save(milestone);+ }++ logger.log("Importing issues from repository " + projectMapping.getGiteaRepo() + "...");+ ImportResult currentResult = importIssues(projectMapping.getGiteaRepo(),+ project, true, option.getIssueImportOption(), users, dryRun, logger);+ result.nonExistentLogins.addAll(currentResult.nonExistentLogins);+ result.nonExistentMilestones.addAll(currentResult.nonExistentMilestones);+ result.unmappedIssueLabels.addAll(currentResult.unmappedIssueLabels);+ result.issuesImported |= currentResult.issuesImported;+ }+ }++ return result.toHtml("Repositories imported successfully");+ } catch (Exception e) {+ for (Long projectId: projectIds)+ OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);+ throw new RuntimeException(e);+ } finally {+ client.close();+ }+ }++ private GlobalIssueSetting getIssueSetting() {+ return OneDev.getInstance(SettingManager.class).getIssueSetting();+ }++ private List<JsonNode> list(Client client, String apiEndpoint, TaskLogger logger) {+ List<JsonNode> result = new ArrayList<>();+ list(client, apiEndpoint, new PageDataConsumer() {++ @Override+ public void consume(List<JsonNode> pageData) {+ result.addAll(pageData);+ }++ }, logger);+ return result;+ }++ private void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer,+ TaskLogger logger) {+ URI uri;+ try {+ uri = new URIBuilder(apiEndpoint)+ .addParameter("limit", String.valueOf(PER_PAGE)).build();+ } catch (URISyntaxException e) {+ throw new RuntimeException(e);+ }++ int page = 1;+ while (true) {+ try {+ URIBuilder builder = new URIBuilder(uri);+ builder.addParameter("page", String.valueOf(page));+ List<JsonNode> pageData = new ArrayList<>();+ for (JsonNode each: JerseyUtils.get(client, builder.build().toString(), logger))+ pageData.add(each);+ pageDataConsumer.consume(pageData);+ if (pageData.size() < PER_PAGE)+ break;+ page++;+ } catch (URISyntaxException|InterruptedException e) {+ throw new RuntimeException(e);+ }+ }+ }++ private Client newClient() {+ Client client = ClientBuilder.newClient();+ client.property(ClientProperties.FOLLOW_REDIRECTS, true);+ client.register(OAuth2ClientSupport.feature(getAccessToken()));+ return client;+ }+@Overridepublic boolean isValid(ConstraintValidatorContext context) {Client client = ClientBuilder.newClient();@@ -111,11 +651,4 @@ public class ImportServer implements Serializable, Validatable {return true;}- Client newClient() {- Client client = ClientBuilder.newClient();- client.property(ClientProperties.FOLLOW_REDIRECTS, true);- client.register(OAuth2ClientSupport.feature(getAccessToken()));- return client;- }-}diff --git a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportUtils.java b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportUtils.javadeleted file mode 100644index 537a0b4bde..0000000000--- a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportUtils.java+++ /dev/null@@ -1,407 +0,0 @@-package io.onedev.server.plugin.imports.gitea;--import java.net.URI;-import java.net.URISyntaxException;-import java.time.Instant;-import java.time.format.DateTimeFormatter;-import java.util.ArrayList;-import java.util.Collection;-import java.util.Date;-import java.util.HashMap;-import java.util.HashSet;-import java.util.LinkedHashMap;-import java.util.LinkedHashSet;-import java.util.List;-import java.util.Map;-import java.util.Optional;-import java.util.Set;-import java.util.concurrent.atomic.AtomicInteger;--import javax.annotation.Nullable;-import javax.ws.rs.client.Client;--import org.apache.commons.lang3.StringUtils;-import org.apache.http.client.utils.URIBuilder;-import org.unbescape.html.HtmlEscape;--import com.fasterxml.jackson.databind.JsonNode;--import io.onedev.commons.utils.ExplicitException;-import io.onedev.commons.utils.TaskLogger;-import io.onedev.server.OneDev;-import io.onedev.server.entitymanager.IssueManager;-import io.onedev.server.entitymanager.SettingManager;-import io.onedev.server.entitymanager.UserManager;-import io.onedev.server.entityreference.ReferenceMigrator;-import io.onedev.server.model.Issue;-import io.onedev.server.model.IssueComment;-import io.onedev.server.model.IssueField;-import io.onedev.server.model.IssueSchedule;-import io.onedev.server.model.Milestone;-import io.onedev.server.model.Project;-import io.onedev.server.model.User;-import io.onedev.server.model.support.LastUpdate;-import io.onedev.server.model.support.administration.GlobalIssueSetting;-import io.onedev.server.model.support.inputspec.InputSpec;-import io.onedev.server.model.support.issue.field.spec.FieldSpec;-import io.onedev.server.persistence.dao.Dao;-import io.onedev.server.util.DateUtils;-import io.onedev.server.util.JerseyUtils;-import io.onedev.server.util.JerseyUtils.PageDataConsumer;-import io.onedev.server.util.Pair;--public class ImportUtils {-- public static final String NAME = "Gitea";-- static final int PER_PAGE = 50;-- static IssueImportOption buildIssueImportOption(ImportServer server, Collection<String> repositories, TaskLogger logger) {- IssueImportOption importOption = new IssueImportOption();- Client client = server.newClient();- try {- Set<String> labels = new LinkedHashSet<>();- for (String repo: repositories) {- String apiEndpoint = server.getApiEndpoint("/repos/" + repo + "/labels");- for (JsonNode labelNode: list(client, apiEndpoint, logger))- labels.add(labelNode.get("name").asText());- try {- apiEndpoint = server.getApiEndpoint("/orgs/" + StringUtils.substringBefore(repo, "/") + "/labels");- for (JsonNode labelNode: list(client, apiEndpoint, logger))- labels.add(labelNode.get("name").asText());- } catch (Exception e) {- // ignore as exception might be thrown if repo belongs to a user account- }- }-- for (String label: labels) {- IssueLabelMapping mapping = new IssueLabelMapping();- mapping.setGiteaIssueLabel(label);- importOption.getIssueLabelMappings().add(mapping);- }- } finally {- client.close();- }- return importOption;- }-- @Nullable- static User getUser(Map<String, Optional<User>> users, JsonNode userNode, TaskLogger logger) {- String login = userNode.get("login").asText();- Optional<User> userOpt = users.get(login);- if (userOpt == null) {- String email = null;- if (userNode.hasNonNull("email"))- email = userNode.get("email").asText(null);- if (email != null)- userOpt = Optional.ofNullable(OneDev.getInstance(UserManager.class).findByVerifiedEmailAddress(email));- else- userOpt = Optional.empty();- users.put(login, userOpt);- }- return userOpt.orElse(null);- }-- static ImportResult importIssues(ImportServer server, String giteaRepo, Project oneDevProject,- boolean useExistingIssueNumbers, IssueImportOption importOption, Map<String, Optional<User>> users,- boolean dryRun, TaskLogger logger) {- Client client = server.newClient();- try {- Set<String> nonExistentMilestones = new HashSet<>();- Set<String> nonExistentLogins = new HashSet<>();- Set<String> unmappedIssueLabels = new HashSet<>();-- Map<String, Pair<FieldSpec, String>> labelMappings = new HashMap<>();- Map<String, Milestone> milestoneMappings = new HashMap<>();-- for (IssueLabelMapping mapping: importOption.getIssueLabelMappings()) {- String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");- String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");- FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);- if (fieldSpec == null)- throw new ExplicitException("No field spec found: " + oneDevFieldName);- labelMappings.put(mapping.getGiteaIssueLabel(), new Pair<>(fieldSpec, oneDevFieldValue));- }-- for (Milestone milestone: oneDevProject.getMilestones())- milestoneMappings.put(milestone.getName(), milestone);-- String initialIssueState = getIssueSetting().getInitialStateSpec().getName();-- List<Issue> issues = new ArrayList<>();-- Map<Long, Long> issueNumberMappings = new HashMap<>();-- AtomicInteger numOfImportedIssues = new AtomicInteger(0);- PageDataConsumer pageDataConsumer = new PageDataConsumer() {-- private String joinAsMultilineHtml(List<String> values) {- List<String> escapedValues = new ArrayList<>();- for (String value: values)- escapedValues.add(HtmlEscape.escapeHtml5(value));- return StringUtils.join(escapedValues, "<br>");- }-- @Override- public void consume(List<JsonNode> pageData) throws InterruptedException {- for (JsonNode issueNode: pageData) {- if (Thread.interrupted())- throw new InterruptedException();-- Map<String, String> extraIssueInfo = new LinkedHashMap<>();-- Issue issue = new Issue();- issue.setProject(oneDevProject);- issue.setTitle(issueNode.get("title").asText());- issue.setDescription(issueNode.get("body").asText(null));-- issue.setNumberScope(oneDevProject.getForkRoot());-- Long oldNumber = issueNode.get("number").asLong();- Long newNumber;- if (dryRun || useExistingIssueNumbers)- newNumber = oldNumber;- else- newNumber = OneDev.getInstance(IssueManager.class).getNextNumber(oneDevProject);- issue.setNumber(newNumber);- issueNumberMappings.put(oldNumber, newNumber);-- String issueFQN = giteaRepo + "#" + oldNumber;-- if (issueNode.get("state").asText().equals("closed"))- issue.setState(importOption.getClosedIssueState());- else- issue.setState(initialIssueState);-- if (issueNode.hasNonNull("milestone")) {- String milestoneName = issueNode.get("milestone").get("title").asText();- Milestone milestone = milestoneMappings.get(milestoneName);- if (milestone != null) {- IssueSchedule schedule = new IssueSchedule();- schedule.setIssue(issue);- schedule.setMilestone(milestone);- issue.getSchedules().add(schedule);- } else {- extraIssueInfo.put("Milestone", milestoneName);- nonExistentMilestones.add(milestoneName);- }- }-- JsonNode userNode = issueNode.get("user");- User user = getUser(users, userNode, logger);- if (user != null) {- issue.setSubmitter(user);- } else {- issue.setSubmitter(OneDev.getInstance(UserManager.class).getUnknown());- nonExistentLogins.add(userNode.get("login").asText());- }-- String created_at = issueNode.get("created_at").asText();- issue.setSubmitDate(Date.from(Instant.from(- DateTimeFormatter.ISO_DATE_TIME.parse(created_at))));-- LastUpdate lastUpdate = new LastUpdate();- lastUpdate.setActivity("Opened");- lastUpdate.setDate(issue.getSubmitDate());- lastUpdate.setUser(issue.getSubmitter());- issue.setLastUpdate(lastUpdate);-- List<JsonNode> assigneeNodes = new ArrayList<>();- if (issueNode.hasNonNull("assignees")) {- for (JsonNode assigneeNode: issueNode.get("assignees"))- assigneeNodes.add(assigneeNode);- } else if (issueNode.hasNonNull("assignee")) {- assigneeNodes.add(issueNode.get("assignee"));- }-- for (JsonNode assigneeNode: assigneeNodes) {- IssueField assigneeField = new IssueField();- assigneeField.setIssue(issue);- assigneeField.setName(importOption.getAssigneesIssueField());- assigneeField.setType(InputSpec.USER);-- user = getUser(users, assigneeNode, logger);- if (user != null) {- assigneeField.setValue(user.getName());- issue.getFields().add(assigneeField);- } else {- nonExistentLogins.add(assigneeNode.get("login").asText());- }- }-- if (importOption.getDueDateIssueField() != null) {- String dueDate = issueNode.get("due_date").asText(null);- if (dueDate != null) {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(importOption.getDueDateIssueField());- issueField.setType(InputSpec.DATE);- issueField.setValue(DateUtils.formatDate(Date.from(- Instant.from(DateTimeFormatter.ISO_DATE_TIME.parse(dueDate)))));- issue.getFields().add(issueField);- }- }-- List<String> currentUnmappedLabels = new ArrayList<>();- for (JsonNode labelNode: issueNode.get("labels")) {- String labelName = labelNode.get("name").asText();- Pair<FieldSpec, String> mapped = labelMappings.get(labelName);- if (mapped != null) {- IssueField labelField = new IssueField();- labelField.setIssue(issue);- labelField.setName(mapped.getFirst().getName());- labelField.setType(InputSpec.ENUMERATION);- labelField.setValue(mapped.getSecond());- labelField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));- issue.getFields().add(labelField);- } else {- currentUnmappedLabels.add(labelName);- unmappedIssueLabels.add(HtmlEscape.escapeHtml5(labelName));- }- }-- if (!currentUnmappedLabels.isEmpty())- extraIssueInfo.put("Labels", joinAsMultilineHtml(currentUnmappedLabels));-- String apiEndpoint = server.getApiEndpoint("/repos/" + giteaRepo- + "/issues/" + oldNumber + "/comments");- for (JsonNode commentNode: list(client, apiEndpoint, logger)) {- String commentContent = commentNode.get("body").asText(null);- if (commentContent != null) {- IssueComment comment = new IssueComment();- comment.setIssue(issue);- comment.setContent(commentContent);-- created_at = commentNode.get("created_at").asText();- comment.setDate(Date.from(Instant.from(- DateTimeFormatter.ISO_DATE_TIME.parse(created_at))));-- userNode = commentNode.get("user");- user = getUser(users, userNode, logger);- if (user != null) {- comment.setUser(user);- } else {- comment.setUser(OneDev.getInstance(UserManager.class).getUnknown());- nonExistentLogins.add(userNode.get("username").asText());- }- issue.getComments().add(comment);- }- }-- issue.setCommentCount(issue.getComments().size());-- Set<String> fieldAndValues = new HashSet<>();- for (IssueField field: issue.getFields()) {- String fieldAndValue = field.getName() + "::" + field.getValue();- if (!fieldAndValues.add(fieldAndValue)) {- String errorMessage = String.format(- "Duplicate issue field mapping (issue: %s, field: %s)",- issueFQN, fieldAndValue);- throw new ExplicitException(errorMessage);- }- }-- if (!extraIssueInfo.isEmpty()) {- StringBuilder builder = new StringBuilder("|");- for (String key: extraIssueInfo.keySet())- builder.append(key).append("|");- builder.append("\n|");- extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));- builder.append("\n|");- for (String value: extraIssueInfo.values())- builder.append(value).append("|");-- if (issue.getDescription() != null)- issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());- else- issue.setDescription(builder.toString());- }- issues.add(issue);- }- logger.log("Imported " + numOfImportedIssues.addAndGet(pageData.size()) + " issues");- }-- };-- String apiEndpoint = server.getApiEndpoint("/repos/" + giteaRepo + "/issues?state=all");- list(client, apiEndpoint, pageDataConsumer, logger);-- if (!dryRun) {- ReferenceMigrator migrator = new ReferenceMigrator(Issue.class, issueNumberMappings);- Dao dao = OneDev.getInstance(Dao.class);- for (Issue issue: issues) {- if (issue.getDescription() != null)- issue.setDescription(migrator.migratePrefixed(issue.getDescription(), "#"));-- OneDev.getInstance(IssueManager.class).save(issue);- for (IssueSchedule schedule: issue.getSchedules())- dao.persist(schedule);- for (IssueField field: issue.getFields())- dao.persist(field);- for (IssueComment comment: issue.getComments()) {- comment.setContent(migrator.migratePrefixed(comment.getContent(), "#"));- dao.persist(comment);- }- }- }-- ImportResult result = new ImportResult();- result.nonExistentLogins.addAll(nonExistentLogins);- result.nonExistentMilestones.addAll(nonExistentMilestones);- result.unmappedIssueLabels.addAll(unmappedIssueLabels);-- if (numOfImportedIssues.get() != 0)- result.issuesImported = true;-- return result;- } finally {- client.close();- }- }-- static GlobalIssueSetting getIssueSetting() {- return OneDev.getInstance(SettingManager.class).getIssueSetting();- }-- static List<JsonNode> list(Client client, String apiEndpoint, TaskLogger logger) {- List<JsonNode> result = new ArrayList<>();- list(client, apiEndpoint, new PageDataConsumer() {-- @Override- public void consume(List<JsonNode> pageData) {- result.addAll(pageData);- }-- }, logger);- return result;- }-- static void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer,- TaskLogger logger) {- URI uri;- try {- uri = new URIBuilder(apiEndpoint)- .addParameter("limit", String.valueOf(PER_PAGE)).build();- } catch (URISyntaxException e) {- throw new RuntimeException(e);- }-- int page = 1;- while (true) {- try {- URIBuilder builder = new URIBuilder(uri);- builder.addParameter("page", String.valueOf(page));- List<JsonNode> pageData = new ArrayList<>();- for (JsonNode each: JerseyUtils.get(client, builder.build().toString(), logger))- pageData.add(each);- pageDataConsumer.consume(pageData);- if (pageData.size() < PER_PAGE)- break;- page++;- } catch (URISyntaxException|InterruptedException e) {- throw new RuntimeException(e);- }- }- }--}diff --git a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/IssueImportSource.java b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/IssueImportSource.javadeleted file mode 100644index f83bae735b..0000000000--- a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/IssueImportSource.java+++ /dev/null@@ -1,70 +0,0 @@-package io.onedev.server.plugin.imports.gitea;--import java.io.Serializable;-import java.util.ArrayList;-import java.util.Collections;-import java.util.List;--import javax.ws.rs.client.Client;--import org.hibernate.validator.constraints.NotEmpty;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--import com.fasterxml.jackson.databind.JsonNode;--import io.onedev.commons.utils.TaskLogger;-import io.onedev.server.web.editable.annotation.ChoiceProvider;-import io.onedev.server.web.editable.annotation.Editable;-import io.onedev.server.web.util.WicketUtils;--@Editable-public class IssueImportSource implements Serializable {-- private static final long serialVersionUID = 1L;-- private static final Logger logger = LoggerFactory.getLogger(IssueImportSource.class);-- private String repository;-- @Editable(order=400, name="Gitea Repository", description="Choose Gitea repository to import issues from")- @ChoiceProvider("getRepositoryChoices")- @NotEmpty- public String getRepository() {- return repository;- }-- public void setRepository(String repository) {- this.repository = repository;- }-- @SuppressWarnings("unused")- private static List<String> getRepositoryChoices() {- List<String> choices = new ArrayList<>();-- ImportServer server = WicketUtils.getPage().getMetaData(ImportServer.META_DATA_KEY);-- Client client = server.newClient();- try {- String apiEndpoint = server.getApiEndpoint("/user/repos");- for (JsonNode repoNode: ImportUtils.list(client, apiEndpoint, new TaskLogger() {-- @Override- public void log(String message, String sessionId) {- logger.info(message);- }-- })) {- String repoName = repoNode.get("name").asText();- String ownerName = repoNode.get("owner").get("login").asText();- choices.add(ownerName + "/" + repoName);- }- } finally {- client.close();- }-- Collections.sort(choices);- return choices;- }--}diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImporter.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImporter.javaindex 72737851ec..0b39273fe4 100644--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImporter.java+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImporter.java@@ -1,10 +1,8 @@package io.onedev.server.plugin.imports.github;-import static io.onedev.server.plugin.imports.github.ImportUtils.NAME;-import static io.onedev.server.plugin.imports.github.ImportUtils.buildImportOption;-import static io.onedev.server.plugin.imports.github.ImportUtils.importIssues;-+import java.io.Serializable;import java.util.HashMap;+import java.util.List;import java.util.Map;import java.util.Optional;@@ -14,35 +12,81 @@ import io.onedev.commons.utils.TaskLogger;import io.onedev.server.imports.IssueImporter;import io.onedev.server.model.Project;import io.onedev.server.model.User;-import io.onedev.server.web.util.WicketUtils;+import io.onedev.server.web.util.ImportStep;-public class GitHubIssueImporter extends IssueImporter<ImportServer, IssueImportSource, IssueImportOption> {+public class GitHubIssueImporter implements IssueImporter {private static final long serialVersionUID = 1L;+ private final ImportStep<ImportServer> serverStep = new ImportStep<ImportServer>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Authenticate to GitHub";+ }++ @Override+ protected ImportServer newSetting() {+ return new ImportServer();+ }++ };++ private final ImportStep<ImportRepository> repositoryStep = new ImportStep<ImportRepository>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Choose repository";+ }++ @Override+ protected ImportRepository newSetting() {+ ImportRepository repository = new ImportRepository();+ repository.server = serverStep.getSetting();+ return repository;+ }++ };++ private final ImportStep<IssueImportOption> optionStep = new ImportStep<IssueImportOption>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Specify import option";+ }++ @Override+ protected IssueImportOption newSetting() {+ return serverStep.getSetting().buildIssueImportOption(+ Lists.newArrayList(repositoryStep.getSetting().getRepository()));+ }++ };+@Overridepublic String getName() {- return NAME;+ return GitHubPluginModule.NAME;}@Override- public String doImport(ImportServer where, IssueImportSource what, IssueImportOption how, Project project,- boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {- logger.log("Importing issues from repository " + what.getRepository() + "...");+ public String doImport(Project project, boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {+ logger.log("Importing issues from repository " + repositoryStep.getSetting().getRepository() + "...");Map<String, Optional<User>> users = new HashMap<>();- return importIssues(where, what.getRepository(), project, retainIssueNumbers, how, users, dryRun, logger)- .toHtml("Issues imported successfully");++ ImportResult result = serverStep.getSetting().importIssues(repositoryStep.getSetting().getRepository(),+ project,retainIssueNumbers, optionStep.getSetting(), users, dryRun, logger);+ return result.toHtml("Issues imported successfully");}@Override- public IssueImportSource getWhat(ImportServer where, TaskLogger logger) {- WicketUtils.getPage().setMetaData(ImportServer.META_DATA_KEY, where);- return new IssueImportSource();+ public List<ImportStep<? extends Serializable>> getSteps() {+ return Lists.newArrayList(serverStep, repositoryStep, optionStep);}- @Override- public IssueImportOption getHow(ImportServer where, IssueImportSource what, TaskLogger logger) {- return buildImportOption(where, Lists.newArrayList(what.getRepository()), logger);- }-}\ No newline at end of filediff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubPluginModule.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubPluginModule.javaindex 6f187c92eb..6cb02e5279 100644--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubPluginModule.java+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubPluginModule.java@@ -1,6 +1,5 @@package io.onedev.server.plugin.imports.github;-import java.io.Serializable;import java.util.Collection;import com.google.common.collect.Lists;@@ -17,6 +16,8 @@ import io.onedev.server.imports.ProjectImporterContribution;*/public class GitHubPluginModule extends AbstractPluginModule {+ static final String NAME = "GitHub";+@Overrideprotected void configure() {super.configure();@@ -24,7 +25,7 @@ public class GitHubPluginModule extends AbstractPluginModule {contribute(ProjectImporterContribution.class, new ProjectImporterContribution() {@Override- public Collection<ProjectImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> getImporters() {+ public Collection<ProjectImporter> getImporters() {return Lists.newArrayList(new GitHubProjectImporter());}@@ -38,7 +39,7 @@ public class GitHubPluginModule extends AbstractPluginModule {contribute(IssueImporterContribution.class, new IssueImporterContribution() {@Override- public Collection<IssueImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> getImporters() {+ public Collection<IssueImporter> getImporters() {return Lists.newArrayList(new GitHubIssueImporter());}diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImporter.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImporter.javaindex 93dd9abb5f..c24c563544 100644--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImporter.java+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImporter.java@@ -1,147 +1,113 @@package io.onedev.server.plugin.imports.github;-import static io.onedev.server.plugin.imports.github.ImportUtils.NAME;-import static io.onedev.server.plugin.imports.github.ImportUtils.get;-import static io.onedev.server.plugin.imports.github.ImportUtils.importIssues;-import static io.onedev.server.plugin.imports.github.ImportUtils.list;--import java.util.ArrayList;-import java.util.Collection;-import java.util.HashMap;+import java.io.Serializable;import java.util.List;-import java.util.Map;-import java.util.Optional;import java.util.stream.Collectors;-import javax.ws.rs.client.Client;--import org.apache.http.client.utils.URIBuilder;-import org.joda.time.format.ISODateTimeFormat;--import com.fasterxml.jackson.databind.JsonNode;-import com.google.common.base.Preconditions;+import com.google.common.collect.Lists;import io.onedev.commons.utils.TaskLogger;-import io.onedev.server.OneDev;-import io.onedev.server.entitymanager.MilestoneManager;-import io.onedev.server.entitymanager.ProjectManager;import io.onedev.server.imports.ProjectImporter;-import io.onedev.server.model.Milestone;-import io.onedev.server.model.Project;-import io.onedev.server.model.User;-import io.onedev.server.storage.StorageManager;+import io.onedev.server.web.util.ImportStep;-public class GitHubProjectImporter extends ProjectImporter<ImportServer, ProjectImportSource, ProjectImportOption> {+public class GitHubProjectImporter implements ProjectImporter {private static final long serialVersionUID = 1L;- @Override- public String getName() {- return NAME;- }+ private final ImportStep<ImportServer> serverStep = new ImportStep<ImportServer>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Authenticate to GitHub";+ }++ @Override+ protected ImportServer newSetting() {+ return new ImportServer();+ }++ };- @Override- public String doImport(ImportServer where, ProjectImportSource what, ProjectImportOption how,- boolean dryRun, TaskLogger logger) {- Collection<Long> projectIds = new ArrayList<>();- Client client = where.newClient();- try {- Map<String, Optional<User>> users = new HashMap<>();- ImportResult result = new ImportResult();- for (ProjectMapping projectMapping: what.getProjectMappings()) {- logger.log("Cloning code from repository " + projectMapping.getGitHubRepo() + "...");-- String apiEndpoint = where.getApiEndpoint("/repos/" + projectMapping.getGitHubRepo());- JsonNode repoNode = get(client, apiEndpoint, logger);- ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);- Project project = projectManager.initialize(projectMapping.getOneDevProject());- Preconditions.checkState(project.isNew());- project.setDescription(repoNode.get("description").asText(null));- project.setIssueManagement(repoNode.get("has_issues").asBoolean());-- boolean isPrivate = repoNode.get("private").asBoolean();- if (!isPrivate && how.getPublicRole() != null)- project.setDefaultRole(how.getPublicRole());-- URIBuilder builder = new URIBuilder(repoNode.get("clone_url").asText());- if (isPrivate)- builder.setUserInfo("git", where.getAccessToken());-- if (!dryRun) {- projectManager.clone(project, builder.build().toString());- projectIds.add(project.getId());- }-- if (how.getIssueImportOption() != null) {- List<Milestone> milestones = new ArrayList<>();- logger.log("Importing milestones from repository " + projectMapping.getGitHubRepo() + "...");- apiEndpoint = where.getApiEndpoint("/repos/" + projectMapping.getGitHubRepo() + "/milestones?state=all");- for (JsonNode milestoneNode: list(client, apiEndpoint, logger)) {- Milestone milestone = new Milestone();- milestone.setName(milestoneNode.get("title").asText());- milestone.setDescription(milestoneNode.get("description").asText(null));- milestone.setProject(project);- String dueDateString = milestoneNode.get("due_on").asText(null);- if (dueDateString != null)- milestone.setDueDate(ISODateTimeFormat.dateTimeNoMillis().parseDateTime(dueDateString).toDate());- if (milestoneNode.get("state").asText().equals("closed"))- milestone.setClosed(true);-- milestones.add(milestone);- project.getMilestones().add(milestone);-- if (!dryRun)- OneDev.getInstance(MilestoneManager.class).save(milestone);- }-- logger.log("Importing issues from repository " + projectMapping.getGitHubRepo() + "...");- ImportResult currentResult = importIssues(where, projectMapping.getGitHubRepo(),- project, true, how.getIssueImportOption(), users, dryRun, logger);- result.nonExistentLogins.addAll(currentResult.nonExistentLogins);- result.nonExistentMilestones.addAll(currentResult.nonExistentMilestones);- result.unmappedIssueLabels.addAll(currentResult.unmappedIssueLabels);-- result.issuesImported |= currentResult.issuesImported;- }- }-- return result.toHtml("Repositories imported successfully");- } catch (Exception e) {- for (Long projectId: projectIds)- OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);- throw new RuntimeException(e);- } finally {- client.close();+ private final ImportStep<ImportOrganization> organizationStep = new ImportStep<ImportOrganization>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Choose organization";}- }- @Override- public ProjectImportSource getWhat(ImportServer where, TaskLogger logger) {- ProjectImportSource importSource = new ProjectImportSource();- Client client = where.newClient();- try {- String apiEndpoint = where.getApiEndpoint("/user/repos");- for (JsonNode repoNode: list(client, apiEndpoint, logger)) {- String repoName = repoNode.get("name").asText();- String ownerName = repoNode.get("owner").get("login").asText();+ @Override+ protected ImportOrganization newSetting() {+ ImportOrganization organization = new ImportOrganization();+ organization.server = serverStep.getSetting();+ return organization;+ }++ };++ private final ImportStep<ImportRepositories> repositoriesStep = new ImportStep<ImportRepositories>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Specify repositories";+ }++ @Override+ protected ImportRepositories newSetting() {+ ImportRepositories repositories = new ImportRepositories();+ String organization = organizationStep.getSetting().getOrganization();+ for (String repository: serverStep.getSetting().listRepositories(organization)) {ProjectMapping projectMapping = new ProjectMapping();- projectMapping.setGitHubRepo(ownerName + "/" + repoName);- projectMapping.setOneDevProject(ownerName + "/" + repoName);- importSource.getProjectMappings().add(projectMapping);- }- } finally {- client.close();+ projectMapping.setGitHubRepo(repository);+ projectMapping.setOneDevProject(repository);+ repositories.getProjectMappings().add(projectMapping);+ }+ return repositories;+ }++ };++ private final ImportStep<ProjectImportOption> optionStep = new ImportStep<ProjectImportOption>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Specify import option";}- return importSource;++ @Override+ protected ProjectImportOption newSetting() {+ ProjectImportOption option = new ProjectImportOption();+ List<String> gitHubRepos = repositoriesStep.getSetting().getProjectMappings().stream()+ .map(it->it.getGitHubRepo()).collect(Collectors.toList());+ option.setIssueImportOption(serverStep.getSetting().buildIssueImportOption(gitHubRepos));+ return option;+ }++ };++ @Override+ public String getName() {+ return GitHubPluginModule.NAME;+ }++ @Override+ public String doImport(boolean dryRun, TaskLogger logger) {+ ImportServer server = serverStep.getSetting();+ ImportRepositories repositories = repositoriesStep.getSetting();+ ProjectImportOption option = optionStep.getSetting();+ return server.importProjects(repositories, option, dryRun, logger);}@Override- public ProjectImportOption getHow(ImportServer where, ProjectImportSource what, TaskLogger logger) {- ProjectImportOption importOption = new ProjectImportOption();- List<String> gitHubRepos = what.getProjectMappings().stream()- .map(it->it.getGitHubRepo()).collect(Collectors.toList());- importOption.setIssueImportOption(ImportUtils.buildImportOption(where, gitHubRepos, logger));- return importOption;+ public List<ImportStep<? extends Serializable>> getSteps() {+ return Lists.newArrayList(serverStep, organizationStep, repositoriesStep, optionStep);}}\ No newline at end of filediff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportOrganization.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportOrganization.javanew file mode 100644index 0000000000..70b4d179e2--- /dev/null+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportOrganization.java@@ -0,0 +1,38 @@+package io.onedev.server.plugin.imports.github;++import java.io.Serializable;+import java.util.List;++import io.onedev.server.util.ComponentContext;+import io.onedev.server.web.editable.BeanEditor;+import io.onedev.server.web.editable.annotation.ChoiceProvider;+import io.onedev.server.web.editable.annotation.Editable;++@Editable+public class ImportOrganization implements Serializable {++ private static final long serialVersionUID = 1L;++ ImportServer server;++ private String organization;++ @Editable(order=100, name="GitHub Organization", description="Select organization to import from. "+ + "Leave empty to import from repositories under current account")+ @ChoiceProvider("getOrganizationChoices")+ public String getOrganization() {+ return organization;+ }++ public void setOrganization(String organization) {+ this.organization = organization;+ }++ @SuppressWarnings("unused")+ private static List<String> getOrganizationChoices() {+ BeanEditor editor = ComponentContext.get().getComponent().findParent(BeanEditor.class);+ ImportOrganization setting = (ImportOrganization) editor.getModelObject();+ return setting.server.listOrganizations();+ }++}diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ProjectImportSource.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportRepositories.javasimilarity index 93%rename from server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ProjectImportSource.javarename to server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportRepositories.javaindex f6c986334b..71c75f5ad9 100644--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ProjectImportSource.java+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportRepositories.java@@ -15,17 +15,19 @@ import io.onedev.server.model.Project;import io.onedev.server.util.validation.Validatable;import io.onedev.server.util.validation.annotation.ClassValidating;import io.onedev.server.web.editable.annotation.Editable;+import io.onedev.server.web.editable.annotation.OmitName;@Editable@ClassValidating-public class ProjectImportSource implements Serializable, Validatable {+public class ImportRepositories implements Serializable, Validatable {private static final long serialVersionUID = 1L;private List<ProjectMapping> projectMappings = new ArrayList<>();- @Editable(order=100, name="Repositories to Import")+ @Editable(order=100)@Size(min=1, max=10000, message="No repositories to import")+ @OmitNamepublic List<ProjectMapping> getProjectMappings() {return projectMappings;}diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportRepository.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportRepository.javanew file mode 100644index 0000000000..7bf699b352--- /dev/null+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportRepository.java@@ -0,0 +1,39 @@+package io.onedev.server.plugin.imports.github;++import java.util.List;++import org.hibernate.validator.constraints.NotEmpty;++import io.onedev.server.util.ComponentContext;+import io.onedev.server.util.EditContext;+import io.onedev.server.web.editable.BeanEditor;+import io.onedev.server.web.editable.annotation.ChoiceProvider;+import io.onedev.server.web.editable.annotation.Editable;++@Editable+public class ImportRepository extends ImportOrganization {++ private static final long serialVersionUID = 1L;++ private String repository;++ @Editable(order=200, name="GitHub Repository", description="Select repository to import from")+ @ChoiceProvider("getRepositoryChoices")+ @NotEmpty+ public String getRepository() {+ return repository;+ }++ public void setRepository(String repository) {+ this.repository = repository;+ }++ @SuppressWarnings("unused")+ private static List<String> getRepositoryChoices() {+ BeanEditor editor = ComponentContext.get().getComponent().findParent(BeanEditor.class);+ ImportRepository setting = (ImportRepository) editor.getModelObject();+ String organization = (String) EditContext.get().getInputValue("organization");+ return setting.server.listRepositories(organization);+ }++}diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportServer.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportServer.javaindex 9b3811bb13..1366d6b75f 100644--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportServer.java+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportServer.java@@ -1,7 +1,21 @@package io.onedev.server.plugin.imports.github;import java.io.Serializable;+import java.net.URI;+import java.net.URISyntaxException;+import java.util.ArrayList;+import java.util.Collection;+import java.util.HashMap;+import java.util.HashSet;+import java.util.LinkedHashMap;+import java.util.LinkedHashSet;+import java.util.List;+import java.util.Map;+import java.util.Optional;+import java.util.Set;+import java.util.concurrent.atomic.AtomicInteger;+import javax.annotation.Nullable;import javax.validation.ConstraintValidatorContext;import javax.ws.rs.client.Client;import javax.ws.rs.client.ClientBuilder;@@ -9,13 +23,45 @@ import javax.ws.rs.client.Invocation;import javax.ws.rs.client.WebTarget;import javax.ws.rs.core.Response;-import org.apache.wicket.MetaDataKey;+import org.apache.http.client.utils.URIBuilder;import org.glassfish.jersey.client.ClientProperties;import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;import org.hibernate.validator.constraints.NotEmpty;+import org.joda.time.format.ISODateTimeFormat;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;+import org.unbescape.html.HtmlEscape;+import com.fasterxml.jackson.databind.JsonNode;+import com.google.common.base.Preconditions;++import edu.emory.mathcs.backport.java.util.Collections;+import io.onedev.commons.utils.ExplicitException;import io.onedev.commons.utils.StringUtils;+import io.onedev.commons.utils.TaskLogger;+import io.onedev.server.OneDev;+import io.onedev.server.entitymanager.IssueManager;+import io.onedev.server.entitymanager.MilestoneManager;+import io.onedev.server.entitymanager.ProjectManager;+import io.onedev.server.entitymanager.SettingManager;+import io.onedev.server.entitymanager.UserManager;+import io.onedev.server.entityreference.ReferenceMigrator;+import io.onedev.server.model.Issue;+import io.onedev.server.model.IssueComment;+import io.onedev.server.model.IssueField;+import io.onedev.server.model.IssueSchedule;+import io.onedev.server.model.Milestone;+import io.onedev.server.model.Project;+import io.onedev.server.model.User;+import io.onedev.server.model.support.LastUpdate;+import io.onedev.server.model.support.administration.GlobalIssueSetting;+import io.onedev.server.model.support.inputspec.InputSpec;+import io.onedev.server.model.support.issue.field.spec.FieldSpec;+import io.onedev.server.persistence.dao.Dao;+import io.onedev.server.storage.StorageManager;import io.onedev.server.util.JerseyUtils;+import io.onedev.server.util.JerseyUtils.PageDataConsumer;+import io.onedev.server.util.Pair;import io.onedev.server.util.validation.Validatable;import io.onedev.server.util.validation.annotation.ClassValidating;import io.onedev.server.web.editable.annotation.Editable;@@ -27,20 +73,18 @@ public class ImportServer implements Serializable, Validatable {private static final long serialVersionUID = 1L;- static final MetaDataKey<ImportServer> META_DATA_KEY = new MetaDataKey<ImportServer>() {-- private static final long serialVersionUID = 1L;-- };+ private static final Logger logger = LoggerFactory.getLogger(ImportServer.class);++ private static final int PER_PAGE = 50;- static final String PROP_API_URL = "apiUrl";+ private static final String PROP_API_URL = "apiUrl";- static final String PROP_ACCESS_TOKEN = "accessToken";+ private static final String PROP_ACCESS_TOKEN = "accessToken";private String apiUrl = "https://api.github.com";private String accessToken;-+@Editable(order=10, name="GitHub API URL", description="Specify GitHub API url, for instance <tt>https://api.github.com</tt>")@NotEmptypublic String getApiUrl() {@@ -63,10 +107,502 @@ public class ImportServer implements Serializable, Validatable {this.accessToken = accessToken;}- public String getApiEndpoint(String apiPath) {+ private String getApiEndpoint(String apiPath) {return StringUtils.stripEnd(apiUrl, "/") + "/" + StringUtils.stripStart(apiPath, "/");}++ List<String> listOrganizations() {+ List<String> organizations = new ArrayList<>();++ Client client = newClient();+ try {+ String apiEndpoint = getApiEndpoint("/user/orgs");+ for (JsonNode orgNode: list(client, apiEndpoint, new TaskLogger() {++ @Override+ public void log(String message, String sessionId) {+ logger.info(message);+ }++ })) {+ organizations.add(orgNode.get("login").asText());+ }+ Collections.sort(organizations);+ } catch (Exception e) {+ logger.error("Error listing organizations", e);+ } finally {+ client.close();+ }++ return organizations;+ }++ List<String> listRepositories(@Nullable String organization) {+ Client client = newClient();+ try {+ String apiEndpoint;+ if (organization != null)+ apiEndpoint = getApiEndpoint("/orgs/" + organization + "/repos");+ else+ apiEndpoint = getApiEndpoint("/user/repos?type=owner");+ List<String> repositories = new ArrayList<>();+ for (JsonNode repoNode: list(client, apiEndpoint, new TaskLogger() {++ @Override+ public void log(String message, String sessionId) {+ logger.info(message);+ }++ })) {+ String repoName = repoNode.get("name").asText();+ String ownerName = repoNode.get("owner").get("login").asText();+ repositories.add(ownerName + "/" + repoName);+ }+ Collections.sort(repositories);+ return repositories;+ } finally {+ client.close();+ }++ }++ private Client newClient() {+ Client client = ClientBuilder.newClient();+ client.property(ClientProperties.FOLLOW_REDIRECTS, true);+ client.register(HttpAuthenticationFeature.basic("git", getAccessToken()));+ return client;+ }++ IssueImportOption buildIssueImportOption(Collection<String> gitHubRepos) {+ IssueImportOption importOption = new IssueImportOption();+ Client client = newClient();+ try {+ TaskLogger taskLogger = new TaskLogger() {++ @Override+ public void log(String message, String sessionId) {+ logger.info(message);+ }++ };+ Set<String> labels = new LinkedHashSet<>();+ for (String each: gitHubRepos) {+ String apiEndpoint = getApiEndpoint("/repos/" + each + "/labels");+ for (JsonNode labelNode: list(client, apiEndpoint, taskLogger))+ labels.add(labelNode.get("name").asText());+ }++ for (String label: labels) {+ IssueLabelMapping mapping = new IssueLabelMapping();+ mapping.setGitHubIssueLabel(label);+ importOption.getIssueLabelMappings().add(mapping);+ }+ } finally {+ client.close();+ }+ return importOption;+ }++ @Nullable+ private User getUser(Client client, Map<String, Optional<User>> users,+ String login, TaskLogger logger) {+ Optional<User> userOpt = users.get(login);+ if (userOpt == null) {+ String apiEndpoint = getApiEndpoint("/users/" + login);+ String email = get(client, apiEndpoint, logger).get("email").asText(null);+ if (email != null)+ userOpt = Optional.ofNullable(OneDev.getInstance(UserManager.class).findByVerifiedEmailAddress(email));+ else+ userOpt = Optional.empty();+ users.put(login, userOpt);+ }+ return userOpt.orElse(null);+ }++ ImportResult importIssues(String gitHubRepo, Project oneDevProject, boolean useExistingIssueNumbers,+ IssueImportOption importOption, Map<String, Optional<User>> users, boolean dryRun, TaskLogger logger) {+ Client client = newClient();+ try {+ Set<String> nonExistentMilestones = new HashSet<>();+ Set<String> nonExistentLogins = new HashSet<>();+ Set<String> unmappedIssueLabels = new HashSet<>();++ Map<String, Pair<FieldSpec, String>> labelMappings = new HashMap<>();+ Map<String, Milestone> milestoneMappings = new HashMap<>();++ for (IssueLabelMapping mapping: importOption.getIssueLabelMappings()) {+ String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");+ String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");+ FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);+ if (fieldSpec == null)+ throw new ExplicitException("No field spec found: " + oneDevFieldName);+ labelMappings.put(mapping.getGitHubIssueLabel(), new Pair<>(fieldSpec, oneDevFieldValue));+ }++ for (Milestone milestone: oneDevProject.getMilestones())+ milestoneMappings.put(milestone.getName(), milestone);++ String initialIssueState = getIssueSetting().getInitialStateSpec().getName();++ List<Issue> issues = new ArrayList<>();++ Map<Long, Long> issueNumberMappings = new HashMap<>();++ AtomicInteger numOfImportedIssues = new AtomicInteger(0);+ PageDataConsumer pageDataConsumer = new PageDataConsumer() {++ private String joinAsMultilineHtml(List<String> values) {+ List<String> escapedValues = new ArrayList<>();+ for (String value: values)+ escapedValues.add(HtmlEscape.escapeHtml5(value));+ return StringUtils.join(escapedValues, "<br>");+ }++ @Override+ public void consume(List<JsonNode> pageData) throws InterruptedException {+ for (JsonNode issueNode: pageData) {+ if (Thread.interrupted())+ throw new InterruptedException();++ if (issueNode.get("pull_request") != null)+ continue;++ Map<String, String> extraIssueInfo = new LinkedHashMap<>();++ Issue issue = new Issue();+ issue.setProject(oneDevProject);+ issue.setTitle(issueNode.get("title").asText());+ issue.setDescription(issueNode.get("body").asText(null));+ issue.setNumberScope(oneDevProject.getForkRoot());+ Long oldNumber = issueNode.get("number").asLong();+ Long newNumber;+ if (dryRun || useExistingIssueNumbers)+ newNumber = oldNumber;+ else+ newNumber = OneDev.getInstance(IssueManager.class).getNextNumber(oneDevProject);+ issue.setNumber(newNumber);+ issueNumberMappings.put(oldNumber, newNumber);++ if (issueNode.get("state").asText().equals("closed"))+ issue.setState(importOption.getClosedIssueState());+ else+ issue.setState(initialIssueState);++ if (issueNode.hasNonNull("milestone")) {+ String milestoneName = issueNode.get("milestone").get("title").asText();+ Milestone milestone = milestoneMappings.get(milestoneName);+ if (milestone != null) {+ IssueSchedule schedule = new IssueSchedule();+ schedule.setIssue(issue);+ schedule.setMilestone(milestone);+ issue.getSchedules().add(schedule);+ } else {+ extraIssueInfo.put("Milestone", milestoneName);+ nonExistentMilestones.add(milestoneName);+ }+ }++ String login = issueNode.get("user").get("login").asText(null);+ User user = getUser(client, users, login, logger);+ if (user != null) {+ issue.setSubmitter(user);+ } else {+ issue.setSubmitter(OneDev.getInstance(UserManager.class).getUnknown());+ nonExistentLogins.add(login);+ }++ issue.setSubmitDate(ISODateTimeFormat.dateTimeNoMillis()+ .parseDateTime(issueNode.get("created_at").asText())+ .toDate());++ LastUpdate lastUpdate = new LastUpdate();+ lastUpdate.setActivity("Opened");+ lastUpdate.setDate(issue.getSubmitDate());+ lastUpdate.setUser(issue.getSubmitter());+ issue.setLastUpdate(lastUpdate);++ for (JsonNode assigneeNode: issueNode.get("assignees")) {+ IssueField assigneeField = new IssueField();+ assigneeField.setIssue(issue);+ assigneeField.setName(importOption.getAssigneesIssueField());+ assigneeField.setType(InputSpec.USER);++ login = assigneeNode.get("login").asText();+ user = getUser(client, users, login, logger);+ if (user != null) {+ assigneeField.setValue(user.getName());+ issue.getFields().add(assigneeField);+ } else {+ nonExistentLogins.add(login);+ }+ }++ String apiEndpoint = getApiEndpoint("/repos/" + gitHubRepo+ + "/issues/" + oldNumber + "/comments");+ for (JsonNode commentNode: list(client, apiEndpoint, logger)) {+ IssueComment comment = new IssueComment();+ comment.setIssue(issue);+ comment.setContent(commentNode.get("body").asText(null));+ comment.setDate(ISODateTimeFormat.dateTimeNoMillis()+ .parseDateTime(commentNode.get("created_at").asText())+ .toDate());++ login = commentNode.get("user").get("login").asText();+ user = getUser(client, users, login, logger);+ if (user != null) {+ comment.setUser(user);+ } else {+ comment.setUser(OneDev.getInstance(UserManager.class).getUnknown());+ nonExistentLogins.add(login);+ }++ issue.getComments().add(comment);+ }++ issue.setCommentCount(issue.getComments().size());++ apiEndpoint = getApiEndpoint("/repos/" + gitHubRepo + "/issues/" + oldNumber + "/labels");+ List<String> currentUnmappedLabels = new ArrayList<>();+ for (JsonNode labelNode: list(client, apiEndpoint, logger)) {+ String labelName = labelNode.get("name").asText();+ Pair<FieldSpec, String> mapped = labelMappings.get(labelName);+ if (mapped != null) {+ IssueField tagField = new IssueField();+ tagField.setIssue(issue);+ tagField.setName(mapped.getFirst().getName());+ tagField.setType(InputSpec.ENUMERATION);+ tagField.setValue(mapped.getSecond());+ tagField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));+ issue.getFields().add(tagField);+ } else {+ currentUnmappedLabels.add(labelName);+ unmappedIssueLabels.add(HtmlEscape.escapeHtml5(labelName));+ }+ }++ if (!currentUnmappedLabels.isEmpty())+ extraIssueInfo.put("Labels", joinAsMultilineHtml(currentUnmappedLabels));++ Set<String> fieldAndValues = new HashSet<>();+ for (IssueField field: issue.getFields()) {+ String fieldAndValue = field.getName() + "::" + field.getValue();+ if (!fieldAndValues.add(fieldAndValue)) {+ String errorMessage = String.format(+ "Duplicate issue field mapping (issue: %s, field: %s)",+ gitHubRepo + "#" + oldNumber, fieldAndValue);+ throw new ExplicitException(errorMessage);+ }+ }++ if (!extraIssueInfo.isEmpty()) {+ StringBuilder builder = new StringBuilder("|");+ for (String key: extraIssueInfo.keySet())+ builder.append(key).append("|");+ builder.append("\n|");+ extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));+ builder.append("\n|");+ for (String value: extraIssueInfo.values())+ builder.append(value).append("|");++ if (issue.getDescription() != null)+ issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());+ else+ issue.setDescription(builder.toString());+ }+ issues.add(issue);+ numOfImportedIssues.incrementAndGet();+ }+ logger.log("Imported " + numOfImportedIssues.get() + " issues");+ }++ };++ String apiEndpoint = getApiEndpoint("/repos/" + gitHubRepo + "/issues?state=all&direction=asc");+ list(client, apiEndpoint, pageDataConsumer, logger);++ if (!dryRun) {+ ReferenceMigrator migrator = new ReferenceMigrator(Issue.class, issueNumberMappings);+ Dao dao = OneDev.getInstance(Dao.class);+ for (Issue issue: issues) {+ if (issue.getDescription() != null)+ issue.setDescription(migrator.migratePrefixed(issue.getDescription(), "#"));++ OneDev.getInstance(IssueManager.class).save(issue);+ for (IssueSchedule schedule: issue.getSchedules())+ dao.persist(schedule);+ for (IssueField field: issue.getFields())+ dao.persist(field);+ for (IssueComment comment: issue.getComments()) {+ comment.setContent(migrator.migratePrefixed(comment.getContent(), "#"));+ dao.persist(comment);+ }+ }+ }++ ImportResult result = new ImportResult();+ result.nonExistentLogins.addAll(nonExistentLogins);+ result.nonExistentMilestones.addAll(nonExistentMilestones);+ result.unmappedIssueLabels.addAll(unmappedIssueLabels);++ if (numOfImportedIssues.get() != 0)+ result.issuesImported = true;++ return result;+ } finally {+ client.close();+ }+ }++ private GlobalIssueSetting getIssueSetting() {+ return OneDev.getInstance(SettingManager.class).getIssueSetting();+ }++ private List<JsonNode> list(Client client, String apiEndpoint, TaskLogger logger) {+ List<JsonNode> result = new ArrayList<>();+ list(client, apiEndpoint, new PageDataConsumer() {++ @Override+ public void consume(List<JsonNode> pageData) {+ result.addAll(pageData);+ }++ }, logger);+ return result;+ }++ private void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer,+ TaskLogger logger) {+ URI uri;+ try {+ uri = new URIBuilder(apiEndpoint)+ .addParameter("per_page", String.valueOf(PER_PAGE)).build();+ } catch (URISyntaxException e) {+ throw new RuntimeException(e);+ }++ int page = 1;+ while (true) {+ try {+ URIBuilder builder = new URIBuilder(uri);+ builder.addParameter("page", String.valueOf(page));+ List<JsonNode> pageData = new ArrayList<>();+ for (JsonNode each: get(client, builder.build().toString(), logger))+ pageData.add(each);+ pageDataConsumer.consume(pageData);+ if (pageData.size() < PER_PAGE)+ break;+ page++;+ } catch (URISyntaxException|InterruptedException e) {+ throw new RuntimeException(e);+ }+ }+ }++ private JsonNode get(Client client, String apiEndpoint, TaskLogger logger) {+ WebTarget target = client.target(apiEndpoint);+ Invocation.Builder builder = target.request();+ while (true) {+ try (Response response = builder.get()) {+ int status = response.getStatus();+ if (status != 200) {+ String errorMessage = response.readEntity(String.class);+ if (StringUtils.isNotBlank(errorMessage)) {+ if (errorMessage.contains("rate limit exceeded")) {+ long resetTime = Long.parseLong(response.getHeaderString("x-ratelimit-reset"))*1000L;+ logger.log("Rate limit exceeded, wait until reset...");+ try {+ Thread.sleep(resetTime + 60*1000L - System.currentTimeMillis());+ continue;+ } catch (InterruptedException e) {+ throw new RuntimeException(e);+ }+ } else {+ throw new ExplicitException(String.format("Http request failed (url: %s, status code: %d, error message: %s)",+ apiEndpoint, status, errorMessage));+ }+ } else {+ throw new ExplicitException(String.format("Http request failed (status: %s)", status));+ }+ }+ return response.readEntity(JsonNode.class);+ }+ }+ }++ String importProjects(ImportRepositories repositories, ProjectImportOption option, boolean dryRun, TaskLogger logger) {+ Collection<Long> projectIds = new ArrayList<>();+ Client client = newClient();+ try {+ Map<String, Optional<User>> users = new HashMap<>();+ ImportResult result = new ImportResult();+ for (ProjectMapping projectMapping: repositories.getProjectMappings()) {+ logger.log("Cloning code from repository " + projectMapping.getGitHubRepo() + "...");++ String apiEndpoint = getApiEndpoint("/repos/" + projectMapping.getGitHubRepo());+ JsonNode repoNode = get(client, apiEndpoint, logger);+ ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);+ Project project = projectManager.initialize(projectMapping.getOneDevProject());+ Preconditions.checkState(project.isNew());+ project.setDescription(repoNode.get("description").asText(null));+ project.setIssueManagement(repoNode.get("has_issues").asBoolean());++ boolean isPrivate = repoNode.get("private").asBoolean();+ if (!isPrivate && option.getPublicRole() != null)+ project.setDefaultRole(option.getPublicRole());++ URIBuilder builder = new URIBuilder(repoNode.get("clone_url").asText());+ if (isPrivate)+ builder.setUserInfo("git", getAccessToken());++ if (!dryRun) {+ projectManager.clone(project, builder.build().toString());+ projectIds.add(project.getId());+ }++ if (option.getIssueImportOption() != null) {+ List<Milestone> milestones = new ArrayList<>();+ logger.log("Importing milestones from repository " + projectMapping.getGitHubRepo() + "...");+ apiEndpoint = getApiEndpoint("/repos/" + projectMapping.getGitHubRepo() + "/milestones?state=all");+ for (JsonNode milestoneNode: list(client, apiEndpoint, logger)) {+ Milestone milestone = new Milestone();+ milestone.setName(milestoneNode.get("title").asText());+ milestone.setDescription(milestoneNode.get("description").asText(null));+ milestone.setProject(project);+ String dueDateString = milestoneNode.get("due_on").asText(null);+ if (dueDateString != null)+ milestone.setDueDate(ISODateTimeFormat.dateTimeNoMillis().parseDateTime(dueDateString).toDate());+ if (milestoneNode.get("state").asText().equals("closed"))+ milestone.setClosed(true);++ milestones.add(milestone);+ project.getMilestones().add(milestone);++ if (!dryRun)+ OneDev.getInstance(MilestoneManager.class).save(milestone);+ }++ logger.log("Importing issues from repository " + projectMapping.getGitHubRepo() + "...");+ ImportResult currentResult = importIssues(projectMapping.getGitHubRepo(),+ project, true, option.getIssueImportOption(), users, dryRun, logger);+ result.nonExistentLogins.addAll(currentResult.nonExistentLogins);+ result.nonExistentMilestones.addAll(currentResult.nonExistentMilestones);+ result.unmappedIssueLabels.addAll(currentResult.unmappedIssueLabels);++ result.issuesImported |= currentResult.issuesImported;+ }+ }++ return result.toHtml("Repositories imported successfully");+ } catch (Exception e) {+ for (Long projectId: projectIds)+ OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);+ throw new RuntimeException(e);+ } finally {+ client.close();+ }+ }+@Overridepublic boolean isValid(ConstraintValidatorContext context) {Client client = ClientBuilder.newClient();@@ -112,11 +648,4 @@ public class ImportServer implements Serializable, Validatable {return true;}- Client newClient() {- Client client = ClientBuilder.newClient();- client.property(ClientProperties.FOLLOW_REDIRECTS, true);- client.register(HttpAuthenticationFeature.basic("git", getAccessToken()));- return client;- }-}diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportUtils.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportUtils.javadeleted file mode 100644index 4c7c6d8276..0000000000--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportUtils.java+++ /dev/null@@ -1,405 +0,0 @@-package io.onedev.server.plugin.imports.github;--import java.net.URI;-import java.net.URISyntaxException;-import java.util.ArrayList;-import java.util.Collection;-import java.util.HashMap;-import java.util.HashSet;-import java.util.LinkedHashMap;-import java.util.LinkedHashSet;-import java.util.List;-import java.util.Map;-import java.util.Optional;-import java.util.Set;-import java.util.concurrent.atomic.AtomicInteger;--import javax.annotation.Nullable;-import javax.ws.rs.client.Client;-import javax.ws.rs.client.Invocation;-import javax.ws.rs.client.WebTarget;-import javax.ws.rs.core.Response;--import org.apache.commons.lang3.StringUtils;-import org.apache.http.client.utils.URIBuilder;-import org.joda.time.format.ISODateTimeFormat;-import org.unbescape.html.HtmlEscape;--import com.fasterxml.jackson.databind.JsonNode;--import io.onedev.commons.utils.ExplicitException;-import io.onedev.commons.utils.TaskLogger;-import io.onedev.server.OneDev;-import io.onedev.server.entitymanager.IssueManager;-import io.onedev.server.entitymanager.SettingManager;-import io.onedev.server.entitymanager.UserManager;-import io.onedev.server.entityreference.ReferenceMigrator;-import io.onedev.server.model.Issue;-import io.onedev.server.model.IssueComment;-import io.onedev.server.model.IssueField;-import io.onedev.server.model.IssueSchedule;-import io.onedev.server.model.Milestone;-import io.onedev.server.model.Project;-import io.onedev.server.model.User;-import io.onedev.server.model.support.LastUpdate;-import io.onedev.server.model.support.administration.GlobalIssueSetting;-import io.onedev.server.model.support.inputspec.InputSpec;-import io.onedev.server.model.support.issue.field.spec.FieldSpec;-import io.onedev.server.persistence.dao.Dao;-import io.onedev.server.util.JerseyUtils.PageDataConsumer;-import io.onedev.server.util.Pair;--public class ImportUtils {-- static final String NAME = "GitHub";-- static final int PER_PAGE = 50;-- static IssueImportOption buildImportOption(ImportServer server, Collection<String> gitHubRepos, TaskLogger logger) {- IssueImportOption importOption = new IssueImportOption();- Client client = server.newClient();- try {- Set<String> labels = new LinkedHashSet<>();- for (String each: gitHubRepos) {- String apiEndpoint = server.getApiEndpoint("/repos/" + each + "/labels");- for (JsonNode labelNode: list(client, apiEndpoint, logger))- labels.add(labelNode.get("name").asText());- }-- for (String label: labels) {- IssueLabelMapping mapping = new IssueLabelMapping();- mapping.setGitHubIssueLabel(label);- importOption.getIssueLabelMappings().add(mapping);- }- } finally {- client.close();- }- return importOption;- }-- @Nullable- static User getUser(Client client, ImportServer importSource,- Map<String, Optional<User>> users, String login, TaskLogger logger) {- Optional<User> userOpt = users.get(login);- if (userOpt == null) {- String apiEndpoint = importSource.getApiEndpoint("/users/" + login);- String email = get(client, apiEndpoint, logger).get("email").asText(null);- if (email != null)- userOpt = Optional.ofNullable(OneDev.getInstance(UserManager.class).findByVerifiedEmailAddress(email));- else- userOpt = Optional.empty();- users.put(login, userOpt);- }- return userOpt.orElse(null);- }-- static ImportResult importIssues(ImportServer server, String gitHubRepo, Project oneDevProject,- boolean useExistingIssueNumbers, IssueImportOption importOption, Map<String, Optional<User>> users,- boolean dryRun, TaskLogger logger) {- Client client = server.newClient();- try {- Set<String> nonExistentMilestones = new HashSet<>();- Set<String> nonExistentLogins = new HashSet<>();- Set<String> unmappedIssueLabels = new HashSet<>();-- Map<String, Pair<FieldSpec, String>> labelMappings = new HashMap<>();- Map<String, Milestone> milestoneMappings = new HashMap<>();-- for (IssueLabelMapping mapping: importOption.getIssueLabelMappings()) {- String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");- String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");- FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);- if (fieldSpec == null)- throw new ExplicitException("No field spec found: " + oneDevFieldName);- labelMappings.put(mapping.getGitHubIssueLabel(), new Pair<>(fieldSpec, oneDevFieldValue));- }-- for (Milestone milestone: oneDevProject.getMilestones())- milestoneMappings.put(milestone.getName(), milestone);-- String initialIssueState = getIssueSetting().getInitialStateSpec().getName();-- List<Issue> issues = new ArrayList<>();-- Map<Long, Long> issueNumberMappings = new HashMap<>();-- AtomicInteger numOfImportedIssues = new AtomicInteger(0);- PageDataConsumer pageDataConsumer = new PageDataConsumer() {-- private String joinAsMultilineHtml(List<String> values) {- List<String> escapedValues = new ArrayList<>();- for (String value: values)- escapedValues.add(HtmlEscape.escapeHtml5(value));- return StringUtils.join(escapedValues, "<br>");- }-- @Override- public void consume(List<JsonNode> pageData) throws InterruptedException {- for (JsonNode issueNode: pageData) {- if (Thread.interrupted())- throw new InterruptedException();-- Map<String, String> extraIssueInfo = new LinkedHashMap<>();-- Issue issue = new Issue();- issue.setProject(oneDevProject);- issue.setTitle(issueNode.get("title").asText());- issue.setDescription(issueNode.get("body").asText(null));- issue.setNumberScope(oneDevProject.getForkRoot());-- Long oldNumber = issueNode.get("number").asLong();- Long newNumber;- if (dryRun || useExistingIssueNumbers)- newNumber = oldNumber;- else- newNumber = OneDev.getInstance(IssueManager.class).getNextNumber(oneDevProject);- issue.setNumber(newNumber);- issueNumberMappings.put(oldNumber, newNumber);-- if (issueNode.get("state").asText().equals("closed"))- issue.setState(importOption.getClosedIssueState());- else- issue.setState(initialIssueState);-- if (issueNode.hasNonNull("milestone")) {- String milestoneName = issueNode.get("milestone").get("title").asText();- Milestone milestone = milestoneMappings.get(milestoneName);- if (milestone != null) {- IssueSchedule schedule = new IssueSchedule();- schedule.setIssue(issue);- schedule.setMilestone(milestone);- issue.getSchedules().add(schedule);- } else {- extraIssueInfo.put("Milestone", milestoneName);- nonExistentMilestones.add(milestoneName);- }- }-- String login = issueNode.get("user").get("login").asText(null);- User user = getUser(client, server, users, login, logger);- if (user != null) {- issue.setSubmitter(user);- } else {- issue.setSubmitter(OneDev.getInstance(UserManager.class).getUnknown());- nonExistentLogins.add(login);- }-- issue.setSubmitDate(ISODateTimeFormat.dateTimeNoMillis()- .parseDateTime(issueNode.get("created_at").asText())- .toDate());-- LastUpdate lastUpdate = new LastUpdate();- lastUpdate.setActivity("Opened");- lastUpdate.setDate(issue.getSubmitDate());- lastUpdate.setUser(issue.getSubmitter());- issue.setLastUpdate(lastUpdate);-- for (JsonNode assigneeNode: issueNode.get("assignees")) {- IssueField assigneeField = new IssueField();- assigneeField.setIssue(issue);- assigneeField.setName(importOption.getAssigneesIssueField());- assigneeField.setType(InputSpec.USER);-- login = assigneeNode.get("login").asText();- user = getUser(client, server, users, login, logger);- if (user != null) {- assigneeField.setValue(user.getName());- issue.getFields().add(assigneeField);- } else {- nonExistentLogins.add(login);- }- }-- String apiEndpoint = server.getApiEndpoint("/repos/" + gitHubRepo- + "/issues/" + oldNumber + "/comments");- for (JsonNode commentNode: list(client, apiEndpoint, logger)) {- IssueComment comment = new IssueComment();- comment.setIssue(issue);- comment.setContent(commentNode.get("body").asText(null));- comment.setDate(ISODateTimeFormat.dateTimeNoMillis()- .parseDateTime(commentNode.get("created_at").asText())- .toDate());-- login = commentNode.get("user").get("login").asText();- user = getUser(client, server, users, login, logger);- if (user != null) {- comment.setUser(user);- } else {- comment.setUser(OneDev.getInstance(UserManager.class).getUnknown());- nonExistentLogins.add(login);- }-- issue.getComments().add(comment);- }-- issue.setCommentCount(issue.getComments().size());-- apiEndpoint = server.getApiEndpoint("/repos/" + gitHubRepo- + "/issues/" + oldNumber + "/labels");- List<String> currentUnmappedLabels = new ArrayList<>();- for (JsonNode labelNode: list(client, apiEndpoint, logger)) {- String labelName = labelNode.get("name").asText();- Pair<FieldSpec, String> mapped = labelMappings.get(labelName);- if (mapped != null) {- IssueField tagField = new IssueField();- tagField.setIssue(issue);- tagField.setName(mapped.getFirst().getName());- tagField.setType(InputSpec.ENUMERATION);- tagField.setValue(mapped.getSecond());- tagField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));- issue.getFields().add(tagField);- } else {- currentUnmappedLabels.add(labelName);- unmappedIssueLabels.add(HtmlEscape.escapeHtml5(labelName));- }- }-- if (!currentUnmappedLabels.isEmpty())- extraIssueInfo.put("Labels", joinAsMultilineHtml(currentUnmappedLabels));-- Set<String> fieldAndValues = new HashSet<>();- for (IssueField field: issue.getFields()) {- String fieldAndValue = field.getName() + "::" + field.getValue();- if (!fieldAndValues.add(fieldAndValue)) {- String errorMessage = String.format(- "Duplicate issue field mapping (issue: %s, field: %s)",- gitHubRepo + "#" + oldNumber, fieldAndValue);- throw new ExplicitException(errorMessage);- }- }-- if (!extraIssueInfo.isEmpty()) {- StringBuilder builder = new StringBuilder("|");- for (String key: extraIssueInfo.keySet())- builder.append(key).append("|");- builder.append("\n|");- extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));- builder.append("\n|");- for (String value: extraIssueInfo.values())- builder.append(value).append("|");-- if (issue.getDescription() != null)- issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());- else- issue.setDescription(builder.toString());- }- issues.add(issue);- }- logger.log("Imported " + numOfImportedIssues.addAndGet(pageData.size()) + " issues");- }-- };-- String apiEndpoint = server.getApiEndpoint("/repos/" + gitHubRepo + "/issues?state=all&direction=asc");- list(client, apiEndpoint, pageDataConsumer, logger);-- if (!dryRun) {- ReferenceMigrator migrator = new ReferenceMigrator(Issue.class, issueNumberMappings);- Dao dao = OneDev.getInstance(Dao.class);- for (Issue issue: issues) {- if (issue.getDescription() != null)- issue.setDescription(migrator.migratePrefixed(issue.getDescription(), "#"));-- OneDev.getInstance(IssueManager.class).save(issue);- for (IssueSchedule schedule: issue.getSchedules())- dao.persist(schedule);- for (IssueField field: issue.getFields())- dao.persist(field);- for (IssueComment comment: issue.getComments()) {- comment.setContent(migrator.migratePrefixed(comment.getContent(), "#"));- dao.persist(comment);- }- }- }-- ImportResult result = new ImportResult();- result.nonExistentLogins.addAll(nonExistentLogins);- result.nonExistentMilestones.addAll(nonExistentMilestones);- result.unmappedIssueLabels.addAll(unmappedIssueLabels);-- if (numOfImportedIssues.get() != 0)- result.issuesImported = true;-- return result;- } finally {- client.close();- }- }-- static GlobalIssueSetting getIssueSetting() {- return OneDev.getInstance(SettingManager.class).getIssueSetting();- }-- static List<JsonNode> list(Client client, String apiEndpoint, TaskLogger logger) {- List<JsonNode> result = new ArrayList<>();- list(client, apiEndpoint, new PageDataConsumer() {-- @Override- public void consume(List<JsonNode> pageData) {- result.addAll(pageData);- }-- }, logger);- return result;- }-- static void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer,- TaskLogger logger) {- URI uri;- try {- uri = new URIBuilder(apiEndpoint)- .addParameter("per_page", String.valueOf(PER_PAGE)).build();- } catch (URISyntaxException e) {- throw new RuntimeException(e);- }-- int page = 1;- while (true) {- try {- URIBuilder builder = new URIBuilder(uri);- builder.addParameter("page", String.valueOf(page));- List<JsonNode> pageData = new ArrayList<>();- for (JsonNode each: get(client, builder.build().toString(), logger))- pageData.add(each);- pageDataConsumer.consume(pageData);- if (pageData.size() < PER_PAGE)- break;- page++;- } catch (URISyntaxException|InterruptedException e) {- throw new RuntimeException(e);- }- }- }-- static JsonNode get(Client client, String apiEndpoint, TaskLogger logger) {- WebTarget target = client.target(apiEndpoint);- Invocation.Builder builder = target.request();- while (true) {- try (Response response = builder.get()) {- int status = response.getStatus();- if (status != 200) {- String errorMessage = response.readEntity(String.class);- if (StringUtils.isNotBlank(errorMessage)) {- if (errorMessage.contains("rate limit exceeded")) {- long resetTime = Long.parseLong(response.getHeaderString("x-ratelimit-reset"))*1000L;- logger.log("Rate limit exceeded, wait until reset...");- try {- Thread.sleep(resetTime + 60*1000L - System.currentTimeMillis());- continue;- } catch (InterruptedException e) {- throw new RuntimeException(e);- }- } else {- throw new ExplicitException(String.format("Http request failed (url: %s, status code: %d, error message: %s)",- apiEndpoint, status, errorMessage));- }- } else {- throw new ExplicitException(String.format("Http request failed (status: %s)", status));- }- }- return response.readEntity(JsonNode.class);- }- }- }--}\ No newline at end of filediff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/IssueImportSource.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/IssueImportSource.javadeleted file mode 100644index 6524544e0f..0000000000--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/IssueImportSource.java+++ /dev/null@@ -1,75 +0,0 @@-package io.onedev.server.plugin.imports.github;--import static io.onedev.server.plugin.imports.github.ImportUtils.list;--import java.io.Serializable;-import java.util.ArrayList;-import java.util.Collections;-import java.util.List;--import javax.ws.rs.client.Client;--import org.hibernate.validator.constraints.NotEmpty;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--import com.fasterxml.jackson.databind.JsonNode;--import io.onedev.commons.utils.TaskLogger;-import io.onedev.server.web.editable.annotation.ChoiceProvider;-import io.onedev.server.web.editable.annotation.Editable;-import io.onedev.server.web.util.WicketUtils;--@Editable-public class IssueImportSource implements Serializable {-- private static final long serialVersionUID = 1L;-- private static final Logger logger = LoggerFactory.getLogger(IssueImportSource.class);-- private String repository;-- @Editable(order=100, name="GitHub Repository", description="Choose GitHub repository to "- + "import issues from. Available repositories will be populated when above options "- + "are specified")- @ChoiceProvider("getRepositoryChoices")- @NotEmpty- public String getRepository() {- return repository;- }-- public void setRepository(String repository) {- this.repository = repository;- }-- @SuppressWarnings("unused")- private static List<String> getRepositoryChoices() {- List<String> choices = new ArrayList<>();-- ImportServer server = WicketUtils.getPage().getMetaData(ImportServer.META_DATA_KEY);-- Client client = server.newClient();- try {- String apiEndpoint = server.getApiEndpoint("/user/repos");- TaskLogger logger = new TaskLogger() {-- @Override- public void log(String message, String sessionId) {- IssueImportSource.logger.info(message);- }-- };- for (JsonNode repoNode: list(client, apiEndpoint, logger)) {- String repoName = repoNode.get("name").asText();- String ownerName = repoNode.get("owner").get("login").asText();- choices.add(ownerName + "/" + repoName);- }- } finally {- client.close();- }-- Collections.sort(choices);- return choices;- }--}diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabIssueImporter.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabIssueImporter.javaindex 67ca0e8397..825255c647 100644--- a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabIssueImporter.java+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabIssueImporter.java@@ -1,9 +1,8 @@package io.onedev.server.plugin.imports.gitlab;-import static io.onedev.server.plugin.imports.gitlab.ImportUtils.NAME;-import static io.onedev.server.plugin.imports.gitlab.ImportUtils.importIssues;-+import java.io.Serializable;import java.util.HashMap;+import java.util.List;import java.util.Map;import java.util.Optional;@@ -13,35 +12,82 @@ import io.onedev.commons.utils.TaskLogger;import io.onedev.server.imports.IssueImporter;import io.onedev.server.model.Project;import io.onedev.server.model.User;-import io.onedev.server.web.util.WicketUtils;+import io.onedev.server.web.util.ImportStep;-public class GitLabIssueImporter extends IssueImporter<ImportServer, IssueImportSource, IssueImportOption> {+public class GitLabIssueImporter implements IssueImporter {private static final long serialVersionUID = 1L;+ private final ImportStep<ImportServer> serverStep = new ImportStep<ImportServer>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Authenticate to GitLab";+ }++ @Override+ protected ImportServer newSetting() {+ return new ImportServer();+ }++ };++ private final ImportStep<ImportProject> projectStep = new ImportStep<ImportProject>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Choose project";+ }++ @Override+ protected ImportProject newSetting() {+ ImportProject repository = new ImportProject();+ repository.server = serverStep.getSetting();+ return repository;+ }++ };++ private final ImportStep<IssueImportOption> optionStep = new ImportStep<IssueImportOption>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Specify import option";+ }++ @Override+ protected IssueImportOption newSetting() {+ return serverStep.getSetting().buildIssueImportOption(+ Lists.newArrayList(projectStep.getSetting().getProject()));+ }++ };+@Overridepublic String getName() {- return NAME;+ return GitLabPluginModule.NAME;}@Override- public String doImport(ImportServer where, IssueImportSource what, IssueImportOption how, Project project,- boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {- logger.log("Importing issues from project " + what.getProject() + "...");+ public String doImport(Project project, boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {+ ImportServer server = serverStep.getSetting();+ String gitLabProject = projectStep.getSetting().getProject();+ IssueImportOption option = optionStep.getSetting();+ logger.log("Importing issues from project " + gitLabProject + "...");Map<String, Optional<User>> users = new HashMap<>();- return importIssues(where, what.getProject(), project, retainIssueNumbers, how, users, dryRun, logger)+ return server.importIssues(gitLabProject, project, retainIssueNumbers, option, users, dryRun, logger).toHtml("Issues imported successfully");}@Override- public IssueImportSource getWhat(ImportServer where, TaskLogger logger) {- WicketUtils.getPage().setMetaData(ImportServer.META_DATA_KEY, where);- return new IssueImportSource();+ public List<ImportStep<? extends Serializable>> getSteps() {+ return Lists.newArrayList(serverStep, projectStep, optionStep);}- @Override- public IssueImportOption getHow(ImportServer where, IssueImportSource what, TaskLogger logger) {- return ImportUtils.buildImportOption(where, Lists.newArrayList(what.getProject()), logger);- }-}\ No newline at end of filediff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabPluginModule.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabPluginModule.javaindex 28cb4dfdc3..f30d856ffe 100644--- a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabPluginModule.java+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabPluginModule.java@@ -1,6 +1,5 @@package io.onedev.server.plugin.imports.gitlab;-import java.io.Serializable;import java.util.Collection;import com.google.common.collect.Lists;@@ -17,14 +16,15 @@ import io.onedev.server.imports.ProjectImporterContribution;*/public class GitLabPluginModule extends AbstractPluginModule {- @Override+ static final String NAME = "GitLab";+protected void configure() {super.configure();contribute(ProjectImporterContribution.class, new ProjectImporterContribution() {@Override- public Collection<ProjectImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> getImporters() {+ public Collection<ProjectImporter> getImporters() {return Lists.newArrayList(new GitLabProjectImporter());}@@ -38,7 +38,7 @@ public class GitLabPluginModule extends AbstractPluginModule {contribute(IssueImporterContribution.class, new IssueImporterContribution() {@Override- public Collection<IssueImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> getImporters() {+ public Collection<IssueImporter> getImporters() {return Lists.newArrayList(new GitLabIssueImporter());}diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabProjectImporter.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabProjectImporter.javaindex 055a40bad5..459c65c489 100644--- a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabProjectImporter.java+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabProjectImporter.java@@ -1,173 +1,111 @@package io.onedev.server.plugin.imports.gitlab;-import static io.onedev.server.plugin.imports.gitlab.ImportUtils.NAME;-import static io.onedev.server.plugin.imports.gitlab.ImportUtils.list;--import java.util.ArrayList;-import java.util.Collection;-import java.util.HashMap;+import java.io.Serializable;import java.util.List;-import java.util.Map;-import java.util.Optional;import java.util.stream.Collectors;-import javax.ws.rs.client.Client;--import org.apache.http.client.utils.URIBuilder;-import org.joda.time.format.ISODateTimeFormat;--import com.fasterxml.jackson.databind.JsonNode;-import com.google.common.base.Preconditions;+import com.google.common.collect.Lists;import io.onedev.commons.utils.TaskLogger;-import io.onedev.server.OneDev;-import io.onedev.server.entitymanager.MilestoneManager;-import io.onedev.server.entitymanager.ProjectManager;import io.onedev.server.imports.ProjectImporter;-import io.onedev.server.model.Milestone;-import io.onedev.server.model.Project;-import io.onedev.server.model.User;-import io.onedev.server.storage.StorageManager;-import io.onedev.server.util.JerseyUtils;+import io.onedev.server.web.util.ImportStep;-public class GitLabProjectImporter extends ProjectImporter<ImportServer, ProjectImportSource, ProjectImportOption> {+public class GitLabProjectImporter implements ProjectImporter {private static final long serialVersionUID = 1L;- @Override- public String getName() {- return NAME;- }-- private List<Milestone> getMilestones(ImportServer server, String groupId, TaskLogger logger) {- Client client = server.newClient();- try {- List<Milestone> milestones = new ArrayList<>();- String apiEndpoint = server.getApiEndpoint("/groups/" + groupId + "/milestones");- for (JsonNode milestoneNode: list(client, apiEndpoint, logger))- milestones.add(getMilestone(milestoneNode));- apiEndpoint = server.getApiEndpoint("/groups/" + groupId);- JsonNode groupNode = JerseyUtils.get(client, apiEndpoint, logger);- JsonNode parentIdNode = groupNode.get("parent_id");- if (parentIdNode != null && parentIdNode.asText(null) != null)- milestones.addAll(getMilestones(server, parentIdNode.asText(), logger));- return milestones;- } finally {- client.close();+ private final ImportStep<ImportServer> serverStep = new ImportStep<ImportServer>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Authenticate to GitLab";}- }++ @Override+ protected ImportServer newSetting() {+ return new ImportServer();+ }++ };- private Milestone getMilestone(JsonNode milestoneNode) {- Milestone milestone = new Milestone();- milestone.setName(milestoneNode.get("title").asText());- milestone.setDescription(milestoneNode.get("description").asText(null));- String dueDateString = milestoneNode.get("due_date").asText(null);- if (dueDateString != null)- milestone.setDueDate(ISODateTimeFormat.date().parseDateTime(dueDateString).toDate());- if (milestoneNode.get("state").asText().equals("closed"))- milestone.setClosed(true);- return milestone;- }+ private final ImportStep<ImportGroup> groupStep = new ImportStep<ImportGroup>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Choose group";+ }++ @Override+ protected ImportGroup newSetting() {+ ImportGroup group = new ImportGroup();+ group.server = serverStep.getSetting();+ return group;+ }++ };- @Override- public String doImport(ImportServer where, ProjectImportSource what, ProjectImportOption how,- boolean dryRun, TaskLogger logger) {- Collection<Long> projectIds = new ArrayList<>();- Client client = where.newClient();- try {- Map<String, Optional<User>> users = new HashMap<>();- ImportResult result = new ImportResult();- for (ProjectMapping projectMapping: what.getProjectMappings()) {- logger.log("Cloning code from project " + projectMapping.getGitLabProject() + "...");-- String apiEndpoint = where.getApiEndpoint("/projects/" + projectMapping.getGitLabProject().replace("/", "%2F"));- JsonNode projectNode = JerseyUtils.get(client, apiEndpoint, logger);- ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);- Project project = projectManager.initialize(projectMapping.getOneDevProject());- Preconditions.checkState(project.isNew());- project.setDescription(projectNode.get("description").asText(null));- project.setIssueManagement(projectNode.get("issues_enabled").asBoolean());-- String visibility = projectNode.get("visibility").asText();- if (!visibility.equals("private") && how.getPublicRole() != null)- project.setDefaultRole(how.getPublicRole());-- URIBuilder builder = new URIBuilder(projectNode.get("http_url_to_repo").asText());- if (!visibility.equals("public"))- builder.setUserInfo("git", where.getAccessToken());-- if (!dryRun) {- projectManager.clone(project, builder.build().toString());- projectIds.add(project.getId());- }-- if (how.getIssueImportOption() != null) {- List<Milestone> milestones = new ArrayList<>();- logger.log("Importing milestones from project " + projectMapping.getGitLabProject() + "...");- apiEndpoint = where.getApiEndpoint("/projects/"- + projectMapping.getGitLabProject().replace("/", "%2F") + "/milestones");- for (JsonNode milestoneNode: list(client, apiEndpoint, logger))- milestones.add(getMilestone(milestoneNode));- JsonNode namespaceNode = projectNode.get("namespace");- if (namespaceNode.get("kind").asText().equals("group")) {- String groupId = namespaceNode.get("id").asText();- milestones.addAll(getMilestones(where, groupId, logger));- }-- for (Milestone milestone: milestones) {- milestone.setProject(project);- project.getMilestones().add(milestone);- if (!dryRun)- OneDev.getInstance(MilestoneManager.class).save(milestone);- }-- logger.log("Importing issues from project " + projectMapping.getGitLabProject() + "...");- ImportResult currentResult = ImportUtils.importIssues(where, projectMapping.getGitLabProject(),- project, true, how.getIssueImportOption(), users, dryRun, logger);- result.nonExistentLogins.addAll(currentResult.nonExistentLogins);- result.nonExistentMilestones.addAll(currentResult.nonExistentMilestones);- result.unmappedIssueLabels.addAll(currentResult.unmappedIssueLabels);- result.tooLargeAttachments.addAll(currentResult.tooLargeAttachments);- result.errorAttachments.addAll(currentResult.errorAttachments);- }- }-- return result.toHtml("Projects imported successfully");- } catch (Exception e) {- for (Long projectId: projectIds)- OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);- throw new RuntimeException(e);- } finally {- client.close();+ private final ImportStep<ImportProjects> projectsStep = new ImportStep<ImportProjects>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Specify projects";}- }- @Override- public ProjectImportSource getWhat(ImportServer where, TaskLogger logger) {- ProjectImportSource importSource = new ProjectImportSource();- Client client = where.newClient();- try {- String apiEndpoint = where.getApiEndpoint("/projects?membership=true");- for (JsonNode projectNode: list(client, apiEndpoint, logger)) {- String pathWithNamespace = projectNode.get("path_with_namespace").asText();+ @Override+ protected ImportProjects newSetting() {+ ImportProjects projects = new ImportProjects();+ String groupId = groupStep.getSetting().getGroupId();+ for (String project: serverStep.getSetting().listProjects(groupId)) {ProjectMapping projectMapping = new ProjectMapping();- projectMapping.setGitLabProject(pathWithNamespace);- projectMapping.setOneDevProject(pathWithNamespace);- importSource.getProjectMappings().add(projectMapping);- }- } finally {- client.close();+ projectMapping.setGitLabProject(project);+ projectMapping.setOneDevProject(project);+ projects.getProjectMappings().add(projectMapping);+ }+ return projects;+ }++ };++ private final ImportStep<ProjectImportOption> optionStep = new ImportStep<ProjectImportOption>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Specify import option";+ }++ @Override+ protected ProjectImportOption newSetting() {+ ProjectImportOption option = new ProjectImportOption();+ List<String> gitLabProjects = projectsStep.getSetting().getProjectMappings().stream()+ .map(it->it.getGitLabProject()).collect(Collectors.toList());+ option.setIssueImportOption(serverStep.getSetting().buildIssueImportOption(gitLabProjects));+ return option;}- return importSource;++ };++ @Override+ public String getName() {+ return GitLabPluginModule.NAME;+ }++ @Override+ public String doImport(boolean dryRun, TaskLogger logger) {+ ImportServer server = serverStep.getSetting();+ return server.importProjects(projectsStep.getSetting(), optionStep.getSetting(), dryRun, logger);}@Override- public ProjectImportOption getHow(ImportServer where, ProjectImportSource what, TaskLogger logger) {- List<String> gitLabProjects = what.getProjectMappings().stream()- .map(it->it.getGitLabProject()).collect(Collectors.toList());- ProjectImportOption importOption = new ProjectImportOption();- importOption.setIssueImportOption(ImportUtils.buildImportOption(where, gitLabProjects, logger));- return importOption;+ public List<ImportStep<? extends Serializable>> getSteps() {+ return Lists.newArrayList(serverStep, groupStep, projectsStep, optionStep);}}\ No newline at end of filediff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportGroup.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportGroup.javanew file mode 100644index 0000000000..fa9a17a605--- /dev/null+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportGroup.java@@ -0,0 +1,38 @@+package io.onedev.server.plugin.imports.gitlab;++import java.io.Serializable;+import java.util.Map;++import io.onedev.server.util.ComponentContext;+import io.onedev.server.web.editable.BeanEditor;+import io.onedev.server.web.editable.annotation.ChoiceProvider;+import io.onedev.server.web.editable.annotation.Editable;++@Editable+public class ImportGroup implements Serializable {++ private static final long serialVersionUID = 1L;++ ImportServer server;++ private String groupId;++ @Editable(order=100, name="GitLab Group", description="Specify group to import from. "+ + "Leave empty to import from projects under current account")+ @ChoiceProvider("getGroupChoices")+ public String getGroupId() {+ return groupId;+ }++ public void setGroupId(String groupId) {+ this.groupId = groupId;+ }++ @SuppressWarnings("unused")+ private static Map<String, String> getGroupChoices() {+ BeanEditor editor = ComponentContext.get().getComponent().findParent(BeanEditor.class);+ ImportGroup setting = (ImportGroup) editor.getModelObject();+ return setting.server.listGroups();+ }++}diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportProject.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportProject.javanew file mode 100644index 0000000000..958e370660--- /dev/null+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportProject.java@@ -0,0 +1,39 @@+package io.onedev.server.plugin.imports.gitlab;++import java.util.List;++import org.hibernate.validator.constraints.NotEmpty;++import io.onedev.server.util.ComponentContext;+import io.onedev.server.util.EditContext;+import io.onedev.server.web.editable.BeanEditor;+import io.onedev.server.web.editable.annotation.ChoiceProvider;+import io.onedev.server.web.editable.annotation.Editable;++@Editable+public class ImportProject extends ImportGroup {++ private static final long serialVersionUID = 1L;++ private String project;++ @Editable(order=200, name="GitLab Project", description="Select project to import from")+ @ChoiceProvider("getProjectChoices")+ @NotEmpty+ public String getProject() {+ return project;+ }++ public void setProject(String project) {+ this.project = project;+ }++ @SuppressWarnings("unused")+ private static List<String> getProjectChoices() {+ BeanEditor editor = ComponentContext.get().getComponent().findParent(BeanEditor.class);+ ImportProject project = (ImportProject) editor.getModelObject();+ String groupId = (String) EditContext.get().getInputValue("groupId");+ return project.server.listProjects(groupId);+ }++}diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ProjectImportSource.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportProjects.javasimilarity index 93%rename from server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ProjectImportSource.javarename to server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportProjects.javaindex a5097c04b2..1b16ab120b 100644--- a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ProjectImportSource.java+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportProjects.java@@ -15,16 +15,18 @@ import io.onedev.server.model.Project;import io.onedev.server.util.validation.Validatable;import io.onedev.server.util.validation.annotation.ClassValidating;import io.onedev.server.web.editable.annotation.Editable;+import io.onedev.server.web.editable.annotation.OmitName;@Editable@ClassValidating-public class ProjectImportSource implements Serializable, Validatable {+public class ImportProjects implements Serializable, Validatable {private static final long serialVersionUID = 1L;private List<ProjectMapping> projectMappings = new ArrayList<>();- @Editable(order=100, name="Projects to Import")+ @Editable(order=100)+ @OmitName@Size(min=1, max=10000, message="No projects to import")public List<ProjectMapping> getProjectMappings() {return projectMappings;diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportServer.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportServer.javaindex fe10e35a14..7d00ba0551 100644--- a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportServer.java+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportServer.java@@ -1,7 +1,25 @@package io.onedev.server.plugin.imports.gitlab;+import java.io.IOException;+import java.io.InputStream;import java.io.Serializable;+import java.net.URI;+import java.net.URISyntaxException;+import java.util.ArrayList;+import java.util.Collection;+import java.util.HashMap;+import java.util.HashSet;+import java.util.LinkedHashMap;+import java.util.LinkedHashSet;+import java.util.List;+import java.util.Map;+import java.util.Optional;+import java.util.Set;+import java.util.concurrent.atomic.AtomicInteger;+import java.util.regex.Matcher;+import java.util.regex.Pattern;+import javax.annotation.Nullable;import javax.validation.ConstraintValidatorContext;import javax.ws.rs.client.Client;import javax.ws.rs.client.ClientBuilder;@@ -9,13 +27,48 @@ import javax.ws.rs.client.Invocation;import javax.ws.rs.client.WebTarget;import javax.ws.rs.core.Response;-import org.apache.wicket.MetaDataKey;+import org.apache.http.client.utils.URIBuilder;import org.glassfish.jersey.client.ClientProperties;import org.glassfish.jersey.client.oauth2.OAuth2ClientSupport;import org.hibernate.validator.constraints.NotEmpty;+import org.joda.time.format.ISODateTimeFormat;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;+import org.unbescape.html.HtmlEscape;+import com.fasterxml.jackson.databind.JsonNode;+import com.google.common.base.Preconditions;++import edu.emory.mathcs.backport.java.util.Collections;+import io.onedev.commons.utils.ExplicitException;import io.onedev.commons.utils.StringUtils;+import io.onedev.commons.utils.TaskLogger;+import io.onedev.server.OneDev;+import io.onedev.server.entitymanager.IssueManager;+import io.onedev.server.entitymanager.MilestoneManager;+import io.onedev.server.entitymanager.ProjectManager;+import io.onedev.server.entitymanager.SettingManager;+import io.onedev.server.entitymanager.UserManager;+import io.onedev.server.entityreference.ReferenceMigrator;+import io.onedev.server.model.Issue;+import io.onedev.server.model.IssueComment;+import io.onedev.server.model.IssueField;+import io.onedev.server.model.IssueSchedule;+import io.onedev.server.model.Milestone;+import io.onedev.server.model.Project;+import io.onedev.server.model.User;+import io.onedev.server.model.support.LastUpdate;+import io.onedev.server.model.support.administration.GlobalIssueSetting;+import io.onedev.server.model.support.inputspec.InputSpec;+import io.onedev.server.model.support.issue.field.spec.FieldSpec;+import io.onedev.server.persistence.dao.Dao;+import io.onedev.server.storage.StorageManager;+import io.onedev.server.util.AttachmentTooLargeException;+import io.onedev.server.util.CollectionUtils;+import io.onedev.server.util.DateUtils;import io.onedev.server.util.JerseyUtils;+import io.onedev.server.util.JerseyUtils.PageDataConsumer;+import io.onedev.server.util.Pair;import io.onedev.server.util.validation.Validatable;import io.onedev.server.util.validation.annotation.ClassValidating;import io.onedev.server.web.editable.annotation.Editable;@@ -27,20 +80,20 @@ public class ImportServer implements Serializable, Validatable {private static final long serialVersionUID = 1L;- static final MetaDataKey<ImportServer> META_DATA_KEY = new MetaDataKey<ImportServer>() {-- private static final long serialVersionUID = 1L;-- };+ private static final Logger logger = LoggerFactory.getLogger(ImportServer.class);++ private static final int PER_PAGE = 50;- static final String PROP_API_URL = "apiUrl";+ private static final Pattern PATTERN_ATTACHMENT = Pattern.compile("\\[(.+?)\\]\\s*\\((/uploads/.+?)\\)");- static final String PROP_ACCESS_TOKEN = "accessToken";+ private static final String PROP_API_URL = "apiUrl";++ private static final String PROP_ACCESS_TOKEN = "accessToken";private String apiUrl;private String accessToken;-+@Editable(order=10, name="GitLab API URL", description="Specify GitLab API url, for instance <tt>https://gitlab.example.com/api/v4</tt>")@NotEmptypublic String getApiUrl() {@@ -62,12 +115,616 @@ public class ImportServer implements Serializable, Validatable {public void setAccessToken(String accessToken) {this.accessToken = accessToken;}++ Map<String, String> listGroups() {+ Map<String, String> groups = new HashMap<>();+ Client client = newClient();+ try {+ String apiEndpoint = getApiEndpoint("/groups?owned=true&top_level_only=true");+ for (JsonNode groupNode: list(client, apiEndpoint, new TaskLogger() {++ @Override+ public void log(String message, String sessionId) {+ logger.info(message);+ }++ })) {+ groups.put(groupNode.get("id").asText(), groupNode.get("full_path").asText());+ }++ } catch (Exception e) {+ logger.error("Error listing groups", e);+ } finally {+ client.close();+ }++ return CollectionUtils.sortByValue(groups);+ }-- public String getApiEndpoint(String apiPath) {+ List<String> listProjects(@Nullable String groupId) {+ List<String> projects = new ArrayList<>();+ Client client = newClient();+ try {+ String apiEndpoint;+ if (groupId != null)+ apiEndpoint = getApiEndpoint("/groups/" + groupId + "/projects?include_subgroups=true");+ else+ apiEndpoint = getApiEndpoint("/projects?owned=true");+ Collection<JsonNode> projectNodes = list(client, apiEndpoint, new TaskLogger() {++ @Override+ public void log(String message, String sessionId) {+ logger.info(message);+ }++ });+ for (JsonNode projectNode: projectNodes) {+ if (groupId != null || projectNode.get("namespace").get("kind").asText().equals("user"))+ projects.add(projectNode.get("path_with_namespace").asText());+ }+ } finally {+ client.close();+ }++ Collections.sort(projects);+ return projects;+ }++ private String getApiEndpoint(String apiPath) {return StringUtils.stripEnd(apiUrl, "/") + "/" + StringUtils.stripStart(apiPath, "/");}+ private Client newClient() {+ Client client = ClientBuilder.newClient();+ client.property(ClientProperties.FOLLOW_REDIRECTS, true);+ client.register(OAuth2ClientSupport.feature(getAccessToken()));+ return client;+ }++ IssueImportOption buildIssueImportOption(Collection<String> gitLabProjects) {+ IssueImportOption option = new IssueImportOption();+ Client client = newClient();+ try {+ TaskLogger taskLogger = new TaskLogger() {++ @Override+ public void log(String message, String sessionId) {+ logger.info(message);+ }++ };+ Set<String> labels = new LinkedHashSet<>();+ for (String each: gitLabProjects) {+ String apiEndpoint = getApiEndpoint("/projects/" + each.replace("/", "%2F") + "/labels");+ for (JsonNode labelNode: list(client, apiEndpoint, taskLogger))+ labels.add(labelNode.get("name").asText());+ }++ for (String label: labels) {+ IssueLabelMapping mapping = new IssueLabelMapping();+ mapping.setGitLabIssueLabel(label);+ option.getIssueLabelMappings().add(mapping);+ }+ } finally {+ client.close();+ }+ return option;+ }++ @Nullable+ private User getUser(Client client, Map<String, Optional<User>> users,+ String userId, TaskLogger logger) {+ Optional<User> userOpt = users.get(userId);+ if (userOpt == null) {+ String apiEndpoint = getApiEndpoint("/users/" + userId);+ JsonNode userNode = JerseyUtils.get(client, apiEndpoint, logger);+ String email = null;+ if (userNode.hasNonNull("email"))+ email = userNode.get("email").asText(null);+ if (email == null && userNode.hasNonNull("public_email"))+ email = userNode.get("public_email").asText(null);+ if (email != null)+ userOpt = Optional.ofNullable(OneDev.getInstance(UserManager.class).findByVerifiedEmailAddress(email));+ else+ userOpt = Optional.empty();+ users.put(userId, userOpt);+ }+ return userOpt.orElse(null);+ }++ private List<Milestone> getMilestones(String groupId, TaskLogger logger) {+ Client client = newClient();+ try {+ List<Milestone> milestones = new ArrayList<>();+ String apiEndpoint = getApiEndpoint("/groups/" + groupId + "/milestones");+ for (JsonNode milestoneNode: list(client, apiEndpoint, logger))+ milestones.add(getMilestone(milestoneNode));+ apiEndpoint = getApiEndpoint("/groups/" + groupId);+ JsonNode groupNode = JerseyUtils.get(client, apiEndpoint, logger);+ JsonNode parentIdNode = groupNode.get("parent_id");+ if (parentIdNode != null && parentIdNode.asText(null) != null)+ milestones.addAll(getMilestones(parentIdNode.asText(), logger));+ return milestones;+ } finally {+ client.close();+ }+ }++ private Milestone getMilestone(JsonNode milestoneNode) {+ Milestone milestone = new Milestone();+ milestone.setName(milestoneNode.get("title").asText());+ milestone.setDescription(milestoneNode.get("description").asText(null));+ String dueDateString = milestoneNode.get("due_date").asText(null);+ if (dueDateString != null)+ milestone.setDueDate(ISODateTimeFormat.date().parseDateTime(dueDateString).toDate());+ if (milestoneNode.get("state").asText().equals("closed"))+ milestone.setClosed(true);+ return milestone;+ }++ String importProjects(ImportProjects projects, ProjectImportOption option, boolean dryRun, TaskLogger logger) {+ Collection<Long> projectIds = new ArrayList<>();+ Client client = newClient();+ try {+ Map<String, Optional<User>> users = new HashMap<>();+ ImportResult result = new ImportResult();+ for (ProjectMapping projectMapping: projects.getProjectMappings()) {+ logger.log("Cloning code from project " + projectMapping.getGitLabProject() + "...");++ String apiEndpoint = getApiEndpoint("/projects/" + projectMapping.getGitLabProject().replace("/", "%2F"));+ JsonNode projectNode = JerseyUtils.get(client, apiEndpoint, logger);+ ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);+ Project project = projectManager.initialize(projectMapping.getOneDevProject());+ Preconditions.checkState(project.isNew());+ project.setDescription(projectNode.get("description").asText(null));+ project.setIssueManagement(projectNode.get("issues_enabled").asBoolean());++ String visibility = projectNode.get("visibility").asText();+ if (!visibility.equals("private") && option.getPublicRole() != null)+ project.setDefaultRole(option.getPublicRole());++ URIBuilder builder = new URIBuilder(projectNode.get("http_url_to_repo").asText());+ if (!visibility.equals("public"))+ builder.setUserInfo("git", getAccessToken());++ if (!dryRun) {+ projectManager.clone(project, builder.build().toString());+ projectIds.add(project.getId());+ }++ if (option.getIssueImportOption() != null) {+ List<Milestone> milestones = new ArrayList<>();+ logger.log("Importing milestones from project " + projectMapping.getGitLabProject() + "...");+ apiEndpoint = getApiEndpoint("/projects/"+ + projectMapping.getGitLabProject().replace("/", "%2F") + "/milestones");+ for (JsonNode milestoneNode: list(client, apiEndpoint, logger))+ milestones.add(getMilestone(milestoneNode));+ JsonNode namespaceNode = projectNode.get("namespace");+ if (namespaceNode.get("kind").asText().equals("group")) {+ String groupId = namespaceNode.get("id").asText();+ milestones.addAll(getMilestones(groupId, logger));+ }++ for (Milestone milestone: milestones) {+ milestone.setProject(project);+ project.getMilestones().add(milestone);+ if (!dryRun)+ OneDev.getInstance(MilestoneManager.class).save(milestone);+ }++ logger.log("Importing issues from project " + projectMapping.getGitLabProject() + "...");+ ImportResult currentResult = importIssues(projectMapping.getGitLabProject(),+ project, true, option.getIssueImportOption(), users, dryRun, logger);+ result.nonExistentLogins.addAll(currentResult.nonExistentLogins);+ result.nonExistentMilestones.addAll(currentResult.nonExistentMilestones);+ result.unmappedIssueLabels.addAll(currentResult.unmappedIssueLabels);+ result.tooLargeAttachments.addAll(currentResult.tooLargeAttachments);+ result.errorAttachments.addAll(currentResult.errorAttachments);+ }+ }++ return result.toHtml("Projects imported successfully");+ } catch (Exception e) {+ for (Long projectId: projectIds)+ OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);+ throw new RuntimeException(e);+ } finally {+ client.close();+ }+ }++ ImportResult importIssues(String gitLabProject, Project oneDevProject,+ boolean useExistingIssueNumbers, IssueImportOption option,+ Map<String, Optional<User>> users, boolean dryRun, TaskLogger logger) {+ Client client = newClient();+ try {+ Set<String> nonExistentMilestones = new HashSet<>();+ Set<String> nonExistentLogins = new HashSet<>();+ Set<String> unmappedIssueLabels = new HashSet<>();+ Set<String> tooLargeAttachments = new LinkedHashSet<>();+ Set<String> errorAttachments = new HashSet<>();++ Map<String, Pair<FieldSpec, String>> labelMappings = new HashMap<>();+ Map<String, Milestone> milestoneMappings = new HashMap<>();++ for (IssueLabelMapping mapping: option.getIssueLabelMappings()) {+ String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");+ String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");+ FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);+ if (fieldSpec == null)+ throw new ExplicitException("No field spec found: " + oneDevFieldName);+ labelMappings.put(mapping.getGitLabIssueLabel(), new Pair<>(fieldSpec, oneDevFieldValue));+ }++ for (Milestone milestone: oneDevProject.getMilestones())+ milestoneMappings.put(milestone.getName(), milestone);++ String initialIssueState = getIssueSetting().getInitialStateSpec().getName();++ List<Issue> issues = new ArrayList<>();++ Map<Long, Long> issueNumberMappings = new HashMap<>();++ AtomicInteger numOfImportedIssues = new AtomicInteger(0);+ PageDataConsumer pageDataConsumer = new PageDataConsumer() {++ @Nullable+ private String processAttachments(String issueUUID, String issueFQN, String markdown,+ String attachmentRootUrl, Set<String> tooLargeAttachments) {+ StringBuffer buffer = new StringBuffer();+ Matcher matcher = PATTERN_ATTACHMENT.matcher(markdown);+ while (matcher.find()) {+ String attachmentUrl = attachmentRootUrl + matcher.group(2);+ String attachmentName = StringUtils.substringAfterLast(attachmentUrl, "/");+ WebTarget target = client.target(attachmentUrl);+ Invocation.Builder builder = target.request();+ try (Response response = builder.get()) {+ String errorMessage = JerseyUtils.checkStatus(attachmentUrl, response);+ if (errorMessage != null) {+ logger.error("Error downloading attachment: " + errorMessage);+ errorAttachments.add(attachmentUrl);+ } else {+ try (InputStream is = response.readEntity(InputStream.class)) {+ String oneDevAttachmentName = oneDevProject.saveAttachment(issueUUID, attachmentName, is);+ String oneDevAttachmentUrl = oneDevProject.getAttachmentUrlPath(issueUUID, oneDevAttachmentName);+ matcher.appendReplacement(buffer,+ Matcher.quoteReplacement("[" + matcher.group(1) + "](" + oneDevAttachmentUrl + ")"));+ } catch (AttachmentTooLargeException ex) {+ tooLargeAttachments.add(issueFQN + ":" + matcher.group(2));+ } catch (IOException e) {+ logger.error("Error downloading attachment", e);+ errorAttachments.add(attachmentUrl);+ }+ }+ }+ }+ matcher.appendTail(buffer);++ return buffer.toString();+ }++ private String joinAsMultilineHtml(List<String> values) {+ List<String> escapedValues = new ArrayList<>();+ for (String value: values)+ escapedValues.add(HtmlEscape.escapeHtml5(value));+ return StringUtils.join(escapedValues, "<br>");+ }++ @Override+ public void consume(List<JsonNode> pageData) throws InterruptedException {+ for (JsonNode issueNode: pageData) {+ if (Thread.interrupted())+ throw new InterruptedException();++ Map<String, String> extraIssueInfo = new LinkedHashMap<>();++ Issue issue = new Issue();+ issue.setProject(oneDevProject);+ issue.setTitle(issueNode.get("title").asText());+ issue.setDescription(issueNode.get("description").asText(null));++ issue.setNumberScope(oneDevProject.getForkRoot());++ Long oldNumber = issueNode.get("iid").asLong();+ Long newNumber;+ if (dryRun || useExistingIssueNumbers)+ newNumber = oldNumber;+ else+ newNumber = OneDev.getInstance(IssueManager.class).getNextNumber(oneDevProject);+ issue.setNumber(newNumber);+ issueNumberMappings.put(oldNumber, newNumber);++ String issueFQN = gitLabProject + "#" + oldNumber;++ if (issueNode.get("state").asText().equals("closed"))+ issue.setState(option.getClosedIssueState());+ else+ issue.setState(initialIssueState);++ if (issueNode.hasNonNull("milestone")) {+ String milestoneName = issueNode.get("milestone").get("title").asText();+ Milestone milestone = milestoneMappings.get(milestoneName);+ if (milestone != null) {+ IssueSchedule schedule = new IssueSchedule();+ schedule.setIssue(issue);+ schedule.setMilestone(milestone);+ issue.getSchedules().add(schedule);+ } else {+ extraIssueInfo.put("Milestone", milestoneName);+ nonExistentMilestones.add(milestoneName);+ }+ }++ JsonNode authorNode = issueNode.get("author");+ User user = getUser(client, users, authorNode.get("id").asText(), logger);+ if (user != null) {+ issue.setSubmitter(user);+ } else {+ issue.setSubmitter(OneDev.getInstance(UserManager.class).getUnknown());+ nonExistentLogins.add(authorNode.get("username").asText());+ }++ issue.setSubmitDate(ISODateTimeFormat.dateTime()+ .parseDateTime(issueNode.get("created_at").asText())+ .toDate());++ LastUpdate lastUpdate = new LastUpdate();+ lastUpdate.setActivity("Opened");+ lastUpdate.setDate(issue.getSubmitDate());+ lastUpdate.setUser(issue.getSubmitter());+ issue.setLastUpdate(lastUpdate);++ List<JsonNode> assigneeNodes = new ArrayList<>();+ if (issueNode.hasNonNull("assignees")) {+ for (JsonNode assigneeNode: issueNode.get("assignees"))+ assigneeNodes.add(assigneeNode);+ } else if (issueNode.hasNonNull("assignee")) {+ assigneeNodes.add(issueNode.get("assignee"));+ }++ for (JsonNode assigneeNode: assigneeNodes) {+ IssueField assigneeField = new IssueField();+ assigneeField.setIssue(issue);+ assigneeField.setName(option.getAssigneesIssueField());+ assigneeField.setType(InputSpec.USER);++ user = getUser(client, users, assigneeNode.get("id").asText(), logger);+ if (user != null) {+ assigneeField.setValue(user.getName());+ issue.getFields().add(assigneeField);+ } else {+ nonExistentLogins.add(assigneeNode.get("username").asText());+ }+ }++ if (option.getDueDateIssueField() != null) {+ String dueDate = issueNode.get("due_date").asText(null);+ if (dueDate != null) {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(option.getDueDateIssueField());+ issueField.setType(InputSpec.DATE);+ issueField.setValue(dueDate);+ issue.getFields().add(issueField);+ }+ }++ JsonNode timeStatsNode = issueNode.get("time_stats");+ if (option.getEstimatedTimeIssueField() != null) {+ int value = timeStatsNode.get("time_estimate").asInt();+ if (value != 0) {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(option.getEstimatedTimeIssueField());+ issueField.setType(InputSpec.WORKING_PERIOD);+ issueField.setValue(DateUtils.formatWorkingPeriod(value/60));+ issue.getFields().add(issueField);+ }+ }+ if (option.getSpentTimeIssueField() != null) {+ int value = timeStatsNode.get("total_time_spent").asInt();+ if (value != 0) {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(option.getSpentTimeIssueField());+ issueField.setType(InputSpec.WORKING_PERIOD);+ issueField.setValue(DateUtils.formatWorkingPeriod(value/60));+ issue.getFields().add(issueField);+ }+ }++ List<String> currentUnmappedLabels = new ArrayList<>();+ for (JsonNode labelNode: issueNode.get("labels")) {+ String labelName = labelNode.asText();+ Pair<FieldSpec, String> mapped = labelMappings.get(labelName);+ if (mapped != null) {+ IssueField labelField = new IssueField();+ labelField.setIssue(issue);+ labelField.setName(mapped.getFirst().getName());+ labelField.setType(InputSpec.ENUMERATION);+ labelField.setValue(mapped.getSecond());+ labelField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));+ issue.getFields().add(labelField);+ } else {+ currentUnmappedLabels.add(labelName);+ unmappedIssueLabels.add(HtmlEscape.escapeHtml5(labelName));+ }+ }++ if (!currentUnmappedLabels.isEmpty())+ extraIssueInfo.put("Labels", joinAsMultilineHtml(currentUnmappedLabels));++ String webUrl = issueNode.get("web_url").asText();+ String attachmentRootUrl = StringUtils.substringBeforeLast(webUrl, "/-");+ if (!dryRun && issue.getDescription() != null) {+ issue.setDescription(processAttachments(issue.getUUID(), issueFQN,+ issue.getDescription(), attachmentRootUrl, tooLargeAttachments));+ }++ String apiEndpoint = getApiEndpoint("/projects/" + gitLabProject.replace("/", "%2F")+ + "/issues/" + oldNumber + "/links");+ Map<String, List<String>> links = new LinkedHashMap<>();+ for (JsonNode linkNode: list(client, apiEndpoint, logger)) {+ String linkIssueNumber = linkNode.get("iid").asText();+ if (linkNode.get("references").get("full").asText().equals(gitLabProject + "#" + linkIssueNumber)) {+ String linkType = linkNode.get("link_type").asText();+ List<String> linksOfType = links.get(linkType);+ if (linksOfType == null) {+ linksOfType = new ArrayList<>();+ links.put(linkType, linksOfType);+ }+ linksOfType.add("#" + linkIssueNumber);+ }+ }+ for (Map.Entry<String, List<String>> entry: links.entrySet())+ extraIssueInfo.put(entry.getKey(), joinAsMultilineHtml(entry.getValue()));++ apiEndpoint = getApiEndpoint("/projects/" + gitLabProject.replace("/", "%2F")+ + "/issues/" + oldNumber + "/notes?sort=asc");+ for (JsonNode noteNode: list(client, apiEndpoint, logger)) {+ if (!noteNode.get("system").asBoolean()) {+ String commentContent = noteNode.get("body").asText(null);+ if (commentContent != null) {+ if (!dryRun) {+ commentContent = processAttachments(issue.getUUID(), issueFQN,+ commentContent, attachmentRootUrl, tooLargeAttachments);+ }++ IssueComment comment = new IssueComment();+ comment.setIssue(issue);+ comment.setContent(commentContent);+ comment.setDate(ISODateTimeFormat.dateTime()+ .parseDateTime(noteNode.get("created_at").asText())+ .toDate());++ authorNode = noteNode.get("author");+ user = getUser(client, users, authorNode.get("id").asText(), logger);+ if (user != null) {+ comment.setUser(user);+ } else {+ comment.setUser(OneDev.getInstance(UserManager.class).getUnknown());+ nonExistentLogins.add(authorNode.get("username").asText());+ }+ issue.getComments().add(comment);+ }+ }+ }++ issue.setCommentCount(issue.getComments().size());++ Set<String> fieldAndValues = new HashSet<>();+ for (IssueField field: issue.getFields()) {+ String fieldAndValue = field.getName() + "::" + field.getValue();+ if (!fieldAndValues.add(fieldAndValue)) {+ String errorMessage = String.format(+ "Duplicate issue field mapping (issue: %s, field: %s)",+ issueFQN, fieldAndValue);+ throw new ExplicitException(errorMessage);+ }+ }++ if (!extraIssueInfo.isEmpty()) {+ StringBuilder builder = new StringBuilder("|");+ for (String key: extraIssueInfo.keySet())+ builder.append(key).append("|");+ builder.append("\n|");+ extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));+ builder.append("\n|");+ for (String value: extraIssueInfo.values())+ builder.append(value).append("|");++ if (issue.getDescription() != null)+ issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());+ else+ issue.setDescription(builder.toString());+ }+ issues.add(issue);+ }+ logger.log("Imported " + numOfImportedIssues.addAndGet(pageData.size()) + " issues");+ }++ };++ String apiEndpoint = getApiEndpoint("/projects/" + gitLabProject.replace("/", "%2F") + "/issues?sort=asc");+ list(client, apiEndpoint, pageDataConsumer, logger);++ if (!dryRun) {+ ReferenceMigrator migrator = new ReferenceMigrator(Issue.class, issueNumberMappings);+ Dao dao = OneDev.getInstance(Dao.class);+ for (Issue issue: issues) {+ if (issue.getDescription() != null)+ issue.setDescription(migrator.migratePrefixed(issue.getDescription(), "#"));++ OneDev.getInstance(IssueManager.class).save(issue);+ for (IssueSchedule schedule: issue.getSchedules())+ dao.persist(schedule);+ for (IssueField field: issue.getFields())+ dao.persist(field);+ for (IssueComment comment: issue.getComments()) {+ comment.setContent(migrator.migratePrefixed(comment.getContent(), "#"));+ dao.persist(comment);+ }+ }+ }++ ImportResult result = new ImportResult();+ result.nonExistentLogins.addAll(nonExistentLogins);+ result.nonExistentMilestones.addAll(nonExistentMilestones);+ result.unmappedIssueLabels.addAll(unmappedIssueLabels);+ result.tooLargeAttachments.addAll(tooLargeAttachments);+ result.errorAttachments.addAll(errorAttachments);++ return result;+ } finally {+ client.close();+ }+ }++ private GlobalIssueSetting getIssueSetting() {+ return OneDev.getInstance(SettingManager.class).getIssueSetting();+ }++ List<JsonNode> list(Client client, String apiEndpoint, TaskLogger logger) {+ List<JsonNode> result = new ArrayList<>();+ list(client, apiEndpoint, new PageDataConsumer() {++ @Override+ public void consume(List<JsonNode> pageData) {+ result.addAll(pageData);+ }++ }, logger);+ return result;+ }++ private void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer,+ TaskLogger logger) {+ URI uri;+ try {+ uri = new URIBuilder(apiEndpoint)+ .addParameter("per_page", String.valueOf(PER_PAGE)).build();+ } catch (URISyntaxException e) {+ throw new RuntimeException(e);+ }++ int page = 1;+ while (true) {+ try {+ URIBuilder builder = new URIBuilder(uri);+ builder.addParameter("page", String.valueOf(page));+ List<JsonNode> pageData = new ArrayList<>();+ for (JsonNode each: JerseyUtils.get(client, builder.build().toString(), logger))+ pageData.add(each);+ pageDataConsumer.consume(pageData);+ if (pageData.size() < PER_PAGE)+ break;+ page++;+ } catch (URISyntaxException|InterruptedException e) {+ throw new RuntimeException(e);+ }+ }+ }+@Overridepublic boolean isValid(ConstraintValidatorContext context) {Client client = ClientBuilder.newClient();@@ -113,11 +770,4 @@ public class ImportServer implements Serializable, Validatable {return true;}- Client newClient() {- Client client = ClientBuilder.newClient();- client.property(ClientProperties.FOLLOW_REDIRECTS, true);- client.register(OAuth2ClientSupport.feature(getAccessToken()));- return client;- }-}diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportUtils.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportUtils.javadeleted file mode 100644index 6dfc1f45f1..0000000000--- a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportUtils.java+++ /dev/null@@ -1,502 +0,0 @@-package io.onedev.server.plugin.imports.gitlab;--import java.io.IOException;-import java.io.InputStream;-import java.net.URI;-import java.net.URISyntaxException;-import java.util.ArrayList;-import java.util.Collection;-import java.util.HashMap;-import java.util.HashSet;-import java.util.LinkedHashMap;-import java.util.LinkedHashSet;-import java.util.List;-import java.util.Map;-import java.util.Optional;-import java.util.Set;-import java.util.concurrent.atomic.AtomicInteger;-import java.util.regex.Matcher;-import java.util.regex.Pattern;--import javax.annotation.Nullable;-import javax.ws.rs.client.Client;-import javax.ws.rs.client.Invocation;-import javax.ws.rs.client.WebTarget;-import javax.ws.rs.core.Response;--import org.apache.commons.lang3.StringUtils;-import org.apache.http.client.utils.URIBuilder;-import org.joda.time.format.ISODateTimeFormat;-import org.unbescape.html.HtmlEscape;--import com.fasterxml.jackson.databind.JsonNode;--import io.onedev.commons.utils.ExplicitException;-import io.onedev.commons.utils.TaskLogger;-import io.onedev.server.OneDev;-import io.onedev.server.entitymanager.IssueManager;-import io.onedev.server.entitymanager.SettingManager;-import io.onedev.server.entitymanager.UserManager;-import io.onedev.server.entityreference.ReferenceMigrator;-import io.onedev.server.model.Issue;-import io.onedev.server.model.IssueComment;-import io.onedev.server.model.IssueField;-import io.onedev.server.model.IssueSchedule;-import io.onedev.server.model.Milestone;-import io.onedev.server.model.Project;-import io.onedev.server.model.User;-import io.onedev.server.model.support.LastUpdate;-import io.onedev.server.model.support.administration.GlobalIssueSetting;-import io.onedev.server.model.support.inputspec.InputSpec;-import io.onedev.server.model.support.issue.field.spec.FieldSpec;-import io.onedev.server.persistence.dao.Dao;-import io.onedev.server.util.AttachmentTooLargeException;-import io.onedev.server.util.DateUtils;-import io.onedev.server.util.JerseyUtils;-import io.onedev.server.util.JerseyUtils.PageDataConsumer;-import io.onedev.server.util.Pair;--public class ImportUtils {-- static final String NAME = "GitLab";-- static final int PER_PAGE = 50;-- private static final Pattern PATTERN_ATTACHMENT = Pattern.compile("\\[(.+?)\\]\\s*\\((/uploads/.+?)\\)");-- static IssueImportOption buildImportOption(ImportServer server, Collection<String> gitLabProjects, TaskLogger logger) {- IssueImportOption importOption = new IssueImportOption();- Client client = server.newClient();- try {- Set<String> labels = new LinkedHashSet<>();- for (String each: gitLabProjects) {- String apiEndpoint = server.getApiEndpoint("/projects/" + each.replace("/", "%2F") + "/labels");- for (JsonNode labelNode: list(client, apiEndpoint, logger))- labels.add(labelNode.get("name").asText());- }-- for (String label: labels) {- IssueLabelMapping mapping = new IssueLabelMapping();- mapping.setGitLabIssueLabel(label);- importOption.getIssueLabelMappings().add(mapping);- }- } finally {- client.close();- }- return importOption;- }-- @Nullable- static User getUser(Client client, ImportServer importSource, Map<String, Optional<User>> users,- String userId, TaskLogger logger) {- Optional<User> userOpt = users.get(userId);- if (userOpt == null) {- String apiEndpoint = importSource.getApiEndpoint("/users/" + userId);- JsonNode userNode = JerseyUtils.get(client, apiEndpoint, logger);- String email = null;- if (userNode.hasNonNull("email"))- email = userNode.get("email").asText(null);- if (email == null && userNode.hasNonNull("public_email"))- email = userNode.get("public_email").asText(null);- if (email != null)- userOpt = Optional.ofNullable(OneDev.getInstance(UserManager.class).findByVerifiedEmailAddress(email));- else- userOpt = Optional.empty();- users.put(userId, userOpt);- }- return userOpt.orElse(null);- }-- static ImportResult importIssues(ImportServer server, String gitLabProject, Project oneDevProject,- boolean useExistingIssueNumbers, IssueImportOption importOption, Map<String, Optional<User>> users,- boolean dryRun, TaskLogger logger) {- Client client = server.newClient();- try {- Set<String> nonExistentMilestones = new HashSet<>();- Set<String> nonExistentLogins = new HashSet<>();- Set<String> unmappedIssueLabels = new HashSet<>();- Set<String> tooLargeAttachments = new LinkedHashSet<>();- Set<String> errorAttachments = new HashSet<>();-- Map<String, Pair<FieldSpec, String>> labelMappings = new HashMap<>();- Map<String, Milestone> milestoneMappings = new HashMap<>();-- for (IssueLabelMapping mapping: importOption.getIssueLabelMappings()) {- String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");- String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");- FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);- if (fieldSpec == null)- throw new ExplicitException("No field spec found: " + oneDevFieldName);- labelMappings.put(mapping.getGitLabIssueLabel(), new Pair<>(fieldSpec, oneDevFieldValue));- }-- for (Milestone milestone: oneDevProject.getMilestones())- milestoneMappings.put(milestone.getName(), milestone);-- String initialIssueState = getIssueSetting().getInitialStateSpec().getName();-- List<Issue> issues = new ArrayList<>();-- Map<Long, Long> issueNumberMappings = new HashMap<>();-- AtomicInteger numOfImportedIssues = new AtomicInteger(0);- PageDataConsumer pageDataConsumer = new PageDataConsumer() {-- @Nullable- private String processAttachments(String issueUUID, String issueFQN, String markdown,- String attachmentRootUrl, Set<String> tooLargeAttachments) {- StringBuffer buffer = new StringBuffer();- Matcher matcher = PATTERN_ATTACHMENT.matcher(markdown);- while (matcher.find()) {- String attachmentUrl = attachmentRootUrl + matcher.group(2);- String attachmentName = StringUtils.substringAfterLast(attachmentUrl, "/");- WebTarget target = client.target(attachmentUrl);- Invocation.Builder builder = target.request();- try (Response response = builder.get()) {- String errorMessage = JerseyUtils.checkStatus(attachmentUrl, response);- if (errorMessage != null) {- logger.error("Error downloading attachment: " + errorMessage);- errorAttachments.add(attachmentUrl);- } else {- try (InputStream is = response.readEntity(InputStream.class)) {- String oneDevAttachmentName = oneDevProject.saveAttachment(issueUUID, attachmentName, is);- String oneDevAttachmentUrl = oneDevProject.getAttachmentUrlPath(issueUUID, oneDevAttachmentName);- matcher.appendReplacement(buffer,- Matcher.quoteReplacement("[" + matcher.group(1) + "](" + oneDevAttachmentUrl + ")"));- } catch (AttachmentTooLargeException ex) {- tooLargeAttachments.add(issueFQN + ":" + matcher.group(2));- } catch (IOException e) {- logger.error("Error downloading attachment", e);- errorAttachments.add(attachmentUrl);- }- }- }- }- matcher.appendTail(buffer);-- return buffer.toString();- }-- private String joinAsMultilineHtml(List<String> values) {- List<String> escapedValues = new ArrayList<>();- for (String value: values)- escapedValues.add(HtmlEscape.escapeHtml5(value));- return StringUtils.join(escapedValues, "<br>");- }-- @Override- public void consume(List<JsonNode> pageData) throws InterruptedException {- for (JsonNode issueNode: pageData) {- if (Thread.interrupted())- throw new InterruptedException();-- Map<String, String> extraIssueInfo = new LinkedHashMap<>();-- Issue issue = new Issue();- issue.setProject(oneDevProject);- issue.setTitle(issueNode.get("title").asText());- issue.setDescription(issueNode.get("description").asText(null));-- issue.setNumberScope(oneDevProject.getForkRoot());-- Long oldNumber = issueNode.get("iid").asLong();- Long newNumber;- if (dryRun || useExistingIssueNumbers)- newNumber = oldNumber;- else- newNumber = OneDev.getInstance(IssueManager.class).getNextNumber(oneDevProject);- issue.setNumber(newNumber);- issueNumberMappings.put(oldNumber, newNumber);-- String issueFQN = gitLabProject + "#" + oldNumber;-- if (issueNode.get("state").asText().equals("closed"))- issue.setState(importOption.getClosedIssueState());- else- issue.setState(initialIssueState);-- if (issueNode.hasNonNull("milestone")) {- String milestoneName = issueNode.get("milestone").get("title").asText();- Milestone milestone = milestoneMappings.get(milestoneName);- if (milestone != null) {- IssueSchedule schedule = new IssueSchedule();- schedule.setIssue(issue);- schedule.setMilestone(milestone);- issue.getSchedules().add(schedule);- } else {- extraIssueInfo.put("Milestone", milestoneName);- nonExistentMilestones.add(milestoneName);- }- }-- JsonNode authorNode = issueNode.get("author");- User user = getUser(client, server, users, authorNode.get("id").asText(), logger);- if (user != null) {- issue.setSubmitter(user);- } else {- issue.setSubmitter(OneDev.getInstance(UserManager.class).getUnknown());- nonExistentLogins.add(authorNode.get("username").asText());- }-- issue.setSubmitDate(ISODateTimeFormat.dateTime()- .parseDateTime(issueNode.get("created_at").asText())- .toDate());-- LastUpdate lastUpdate = new LastUpdate();- lastUpdate.setActivity("Opened");- lastUpdate.setDate(issue.getSubmitDate());- lastUpdate.setUser(issue.getSubmitter());- issue.setLastUpdate(lastUpdate);-- List<JsonNode> assigneeNodes = new ArrayList<>();- if (issueNode.hasNonNull("assignees")) {- for (JsonNode assigneeNode: issueNode.get("assignees"))- assigneeNodes.add(assigneeNode);- } else if (issueNode.hasNonNull("assignee")) {- assigneeNodes.add(issueNode.get("assignee"));- }-- for (JsonNode assigneeNode: assigneeNodes) {- IssueField assigneeField = new IssueField();- assigneeField.setIssue(issue);- assigneeField.setName(importOption.getAssigneesIssueField());- assigneeField.setType(InputSpec.USER);-- user = getUser(client, server, users, assigneeNode.get("id").asText(), logger);- if (user != null) {- assigneeField.setValue(user.getName());- issue.getFields().add(assigneeField);- } else {- nonExistentLogins.add(assigneeNode.get("username").asText());- }- }-- if (importOption.getDueDateIssueField() != null) {- String dueDate = issueNode.get("due_date").asText(null);- if (dueDate != null) {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(importOption.getDueDateIssueField());- issueField.setType(InputSpec.DATE);- issueField.setValue(dueDate);- issue.getFields().add(issueField);- }- }-- JsonNode timeStatsNode = issueNode.get("time_stats");- if (importOption.getEstimatedTimeIssueField() != null) {- int value = timeStatsNode.get("time_estimate").asInt();- if (value != 0) {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(importOption.getEstimatedTimeIssueField());- issueField.setType(InputSpec.WORKING_PERIOD);- issueField.setValue(DateUtils.formatWorkingPeriod(value/60));- issue.getFields().add(issueField);- }- }- if (importOption.getSpentTimeIssueField() != null) {- int value = timeStatsNode.get("total_time_spent").asInt();- if (value != 0) {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(importOption.getSpentTimeIssueField());- issueField.setType(InputSpec.WORKING_PERIOD);- issueField.setValue(DateUtils.formatWorkingPeriod(value/60));- issue.getFields().add(issueField);- }- }-- List<String> currentUnmappedLabels = new ArrayList<>();- for (JsonNode labelNode: issueNode.get("labels")) {- String labelName = labelNode.asText();- Pair<FieldSpec, String> mapped = labelMappings.get(labelName);- if (mapped != null) {- IssueField labelField = new IssueField();- labelField.setIssue(issue);- labelField.setName(mapped.getFirst().getName());- labelField.setType(InputSpec.ENUMERATION);- labelField.setValue(mapped.getSecond());- labelField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));- issue.getFields().add(labelField);- } else {- currentUnmappedLabels.add(labelName);- unmappedIssueLabels.add(HtmlEscape.escapeHtml5(labelName));- }- }-- if (!currentUnmappedLabels.isEmpty())- extraIssueInfo.put("Labels", joinAsMultilineHtml(currentUnmappedLabels));-- String webUrl = issueNode.get("web_url").asText();- String attachmentRootUrl = StringUtils.substringBeforeLast(webUrl, "/-");- if (!dryRun && issue.getDescription() != null) {- issue.setDescription(processAttachments(issue.getUUID(), issueFQN,- issue.getDescription(), attachmentRootUrl, tooLargeAttachments));- }-- String apiEndpoint = server.getApiEndpoint("/projects/" + gitLabProject.replace("/", "%2F")- + "/issues/" + oldNumber + "/links");- Map<String, List<String>> links = new LinkedHashMap<>();- for (JsonNode linkNode: list(client, apiEndpoint, logger)) {- String linkIssueNumber = linkNode.get("iid").asText();- if (linkNode.get("references").get("full").asText().equals(gitLabProject + "#" + linkIssueNumber)) {- String linkType = linkNode.get("link_type").asText();- List<String> linksOfType = links.get(linkType);- if (linksOfType == null) {- linksOfType = new ArrayList<>();- links.put(linkType, linksOfType);- }- linksOfType.add("#" + linkIssueNumber);- }- }- for (Map.Entry<String, List<String>> entry: links.entrySet())- extraIssueInfo.put(entry.getKey(), joinAsMultilineHtml(entry.getValue()));-- apiEndpoint = server.getApiEndpoint("/projects/" + gitLabProject.replace("/", "%2F")- + "/issues/" + oldNumber + "/notes?sort=asc");- for (JsonNode noteNode: list(client, apiEndpoint, logger)) {- if (!noteNode.get("system").asBoolean()) {- String commentContent = noteNode.get("body").asText(null);- if (commentContent != null) {- if (!dryRun) {- commentContent = processAttachments(issue.getUUID(), issueFQN,- commentContent, attachmentRootUrl, tooLargeAttachments);- }-- IssueComment comment = new IssueComment();- comment.setIssue(issue);- comment.setContent(commentContent);- comment.setDate(ISODateTimeFormat.dateTime()- .parseDateTime(noteNode.get("created_at").asText())- .toDate());-- authorNode = noteNode.get("author");- user = getUser(client, server, users, authorNode.get("id").asText(), logger);- if (user != null) {- comment.setUser(user);- } else {- comment.setUser(OneDev.getInstance(UserManager.class).getUnknown());- nonExistentLogins.add(authorNode.get("username").asText());- }- issue.getComments().add(comment);- }- }- }-- issue.setCommentCount(issue.getComments().size());-- Set<String> fieldAndValues = new HashSet<>();- for (IssueField field: issue.getFields()) {- String fieldAndValue = field.getName() + "::" + field.getValue();- if (!fieldAndValues.add(fieldAndValue)) {- String errorMessage = String.format(- "Duplicate issue field mapping (issue: %s, field: %s)",- issueFQN, fieldAndValue);- throw new ExplicitException(errorMessage);- }- }-- if (!extraIssueInfo.isEmpty()) {- StringBuilder builder = new StringBuilder("|");- for (String key: extraIssueInfo.keySet())- builder.append(key).append("|");- builder.append("\n|");- extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));- builder.append("\n|");- for (String value: extraIssueInfo.values())- builder.append(value).append("|");-- if (issue.getDescription() != null)- issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());- else- issue.setDescription(builder.toString());- }- issues.add(issue);- }- logger.log("Imported " + numOfImportedIssues.addAndGet(pageData.size()) + " issues");- }-- };-- String apiEndpoint = server.getApiEndpoint("/projects/" + gitLabProject.replace("/", "%2F") + "/issues?sort=asc");- list(client, apiEndpoint, pageDataConsumer, logger);-- if (!dryRun) {- ReferenceMigrator migrator = new ReferenceMigrator(Issue.class, issueNumberMappings);- Dao dao = OneDev.getInstance(Dao.class);- for (Issue issue: issues) {- if (issue.getDescription() != null)- issue.setDescription(migrator.migratePrefixed(issue.getDescription(), "#"));-- OneDev.getInstance(IssueManager.class).save(issue);- for (IssueSchedule schedule: issue.getSchedules())- dao.persist(schedule);- for (IssueField field: issue.getFields())- dao.persist(field);- for (IssueComment comment: issue.getComments()) {- comment.setContent(migrator.migratePrefixed(comment.getContent(), "#"));- dao.persist(comment);- }- }- }-- ImportResult result = new ImportResult();- result.nonExistentLogins.addAll(nonExistentLogins);- result.nonExistentMilestones.addAll(nonExistentMilestones);- result.unmappedIssueLabels.addAll(unmappedIssueLabels);- result.tooLargeAttachments.addAll(tooLargeAttachments);- result.errorAttachments.addAll(errorAttachments);-- return result;- } finally {- client.close();- }- }-- static GlobalIssueSetting getIssueSetting() {- return OneDev.getInstance(SettingManager.class).getIssueSetting();- }-- static List<JsonNode> list(Client client, String apiEndpoint, TaskLogger logger) {- List<JsonNode> result = new ArrayList<>();- list(client, apiEndpoint, new PageDataConsumer() {-- @Override- public void consume(List<JsonNode> pageData) {- result.addAll(pageData);- }-- }, logger);- return result;- }-- static void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer,- TaskLogger logger) {- URI uri;- try {- uri = new URIBuilder(apiEndpoint)- .addParameter("per_page", String.valueOf(PER_PAGE)).build();- } catch (URISyntaxException e) {- throw new RuntimeException(e);- }-- int page = 1;- while (true) {- try {- URIBuilder builder = new URIBuilder(uri);- builder.addParameter("page", String.valueOf(page));- List<JsonNode> pageData = new ArrayList<>();- for (JsonNode each: JerseyUtils.get(client, builder.build().toString(), logger))- pageData.add(each);- pageDataConsumer.consume(pageData);- if (pageData.size() < PER_PAGE)- break;- page++;- } catch (URISyntaxException|InterruptedException e) {- throw new RuntimeException(e);- }- }- }--}\ No newline at end of filediff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/IssueImportSource.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/IssueImportSource.javadeleted file mode 100644index 3647471264..0000000000--- a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/IssueImportSource.java+++ /dev/null@@ -1,69 +0,0 @@-package io.onedev.server.plugin.imports.gitlab;--import java.io.Serializable;-import java.util.ArrayList;-import java.util.Collection;-import java.util.Collections;-import java.util.List;--import javax.ws.rs.client.Client;--import org.hibernate.validator.constraints.NotEmpty;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--import com.fasterxml.jackson.databind.JsonNode;--import io.onedev.commons.utils.TaskLogger;-import io.onedev.server.web.editable.annotation.ChoiceProvider;-import io.onedev.server.web.editable.annotation.Editable;-import io.onedev.server.web.util.WicketUtils;--@Editable-public class IssueImportSource implements Serializable {-- private static final long serialVersionUID = 1L;-- private static final Logger logger = LoggerFactory.getLogger(IssueImportSource.class);-- private String project;-- @Editable(order=400, name="GitLab Project", description="Choose GitLab project to import issues from")- @ChoiceProvider("getProjectChoices")- @NotEmpty- public String getProject() {- return project;- }-- public void setProject(String project) {- this.project = project;- }-- @SuppressWarnings("unused")- private static List<String> getProjectChoices() {- List<String> choices = new ArrayList<>();-- ImportServer server = WicketUtils.getPage().getMetaData(ImportServer.META_DATA_KEY);-- Client client = server.newClient();- try {- String apiEndpoint = server.getApiEndpoint("/projects?membership=true");- Collection<JsonNode> projectNodes = ImportUtils.list(client, apiEndpoint, new TaskLogger() {-- @Override- public void log(String message, String sessionId) {- logger.info(message);- }-- });- for (JsonNode projectNode: projectNodes)- choices.add(projectNode.get("path_with_namespace").asText());- } finally {- client.close();- }-- Collections.sort(choices);- return choices;- }--}diff --git a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportProject.java b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportProject.javanew file mode 100644index 0000000000..29f08275fd--- /dev/null+++ b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportProject.java@@ -0,0 +1,40 @@+package io.onedev.server.plugin.imports.jiracloud;++import java.io.Serializable;+import java.util.List;++import org.hibernate.validator.constraints.NotEmpty;++import io.onedev.server.util.ComponentContext;+import io.onedev.server.web.editable.BeanEditor;+import io.onedev.server.web.editable.annotation.ChoiceProvider;+import io.onedev.server.web.editable.annotation.Editable;++@Editable+public class ImportProject implements Serializable {++ private static final long serialVersionUID = 1L;++ ImportServer server;++ private String project;++ @Editable(order=400, name="JIRA Project", description="Choose JIRA project to import issues from")+ @ChoiceProvider("getProjectChoices")+ @NotEmpty+ public String getProject() {+ return project;+ }++ public void setProject(String project) {+ this.project = project;+ }++ @SuppressWarnings("unused")+ private static List<String> getProjectChoices() {+ BeanEditor editor = ComponentContext.get().getComponent().findParent(BeanEditor.class);+ ImportProject setting = (ImportProject) editor.getModelObject();+ return setting.server.listProjects();+ }++}diff --git a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ProjectImportSource.java b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportProjects.javasimilarity index 94%rename from server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ProjectImportSource.javarename to server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportProjects.javaindex f6c09c8339..ea95f4d782 100644--- a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ProjectImportSource.java+++ b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportProjects.java@@ -15,16 +15,18 @@ import io.onedev.server.model.Project;import io.onedev.server.util.validation.Validatable;import io.onedev.server.util.validation.annotation.ClassValidating;import io.onedev.server.web.editable.annotation.Editable;+import io.onedev.server.web.editable.annotation.OmitName;@Editable@ClassValidating-public class ProjectImportSource implements Serializable, Validatable {+public class ImportProjects implements Serializable, Validatable {private static final long serialVersionUID = 1L;private List<ProjectMapping> projectMappings = new ArrayList<>();@Editable(order=100, name="Projects to Import")+ @OmitName@Size(min=1, max=10000, message="No projects to import")public List<ProjectMapping> getProjectMappings() {return projectMappings;diff --git a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportServer.java b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportServer.javaindex 6248fcf98b..c8f1c4a8e6 100644--- a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportServer.java+++ b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportServer.java@@ -1,7 +1,28 @@package io.onedev.server.plugin.imports.jiracloud;+import java.io.IOException;+import java.io.InputStream;import java.io.Serializable;+import java.io.UnsupportedEncodingException;+import java.net.URI;+import java.net.URISyntaxException;+import java.net.URLEncoder;+import java.nio.charset.StandardCharsets;+import java.util.ArrayList;+import java.util.Collection;+import java.util.Collections;+import java.util.Date;+import java.util.HashMap;+import java.util.HashSet;+import java.util.LinkedHashMap;+import java.util.LinkedHashSet;+import java.util.List;+import java.util.Map;+import java.util.Optional;+import java.util.Set;+import java.util.concurrent.atomic.AtomicInteger;+import javax.annotation.Nullable;import javax.validation.ConstraintValidatorContext;import javax.ws.rs.client.Client;import javax.ws.rs.client.ClientBuilder;@@ -10,14 +31,46 @@ import javax.ws.rs.client.WebTarget;import javax.ws.rs.core.MediaType;import javax.ws.rs.core.Response;-import org.apache.wicket.MetaDataKey;+import org.apache.http.client.utils.URIBuilder;import org.glassfish.jersey.client.ClientProperties;import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;import org.hibernate.validator.constraints.Email;import org.hibernate.validator.constraints.NotEmpty;+import org.joda.time.format.ISODateTimeFormat;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;+import org.unbescape.html.HtmlEscape;+import com.fasterxml.jackson.databind.JsonNode;+import com.google.common.base.Joiner;+import com.google.common.base.Preconditions;+import com.google.common.base.Splitter;++import io.onedev.commons.utils.ExplicitException;import io.onedev.commons.utils.StringUtils;+import io.onedev.commons.utils.TaskLogger;+import io.onedev.server.OneDev;+import io.onedev.server.entitymanager.IssueManager;+import io.onedev.server.entitymanager.ProjectManager;+import io.onedev.server.entitymanager.SettingManager;+import io.onedev.server.entitymanager.UserManager;+import io.onedev.server.entityreference.ReferenceMigrator;+import io.onedev.server.model.Issue;+import io.onedev.server.model.IssueComment;+import io.onedev.server.model.IssueField;+import io.onedev.server.model.IssueSchedule;+import io.onedev.server.model.Project;+import io.onedev.server.model.User;+import io.onedev.server.model.support.LastUpdate;+import io.onedev.server.model.support.administration.GlobalIssueSetting;+import io.onedev.server.model.support.inputspec.InputSpec;+import io.onedev.server.model.support.issue.field.spec.FieldSpec;+import io.onedev.server.persistence.dao.Dao;+import io.onedev.server.storage.StorageManager;+import io.onedev.server.util.DateUtils;import io.onedev.server.util.JerseyUtils;+import io.onedev.server.util.JerseyUtils.PageDataConsumer;+import io.onedev.server.util.Pair;import io.onedev.server.util.validation.Validatable;import io.onedev.server.util.validation.annotation.ClassValidating;import io.onedev.server.web.editable.annotation.Editable;@@ -29,13 +82,11 @@ public class ImportServer implements Serializable, Validatable {private static final long serialVersionUID = 1L;- static final MetaDataKey<ImportServer> META_DATA_KEY = new MetaDataKey<ImportServer>() {-- private static final long serialVersionUID = 1L;-- };+ private static final Logger logger = LoggerFactory.getLogger(ImportServer.class);++ private static final int PER_PAGE = 50;- static final String PROP_API_URL = "apiUrl";+ private static final String PROP_API_URL = "apiUrl";private String apiUrl;@@ -75,10 +126,869 @@ public class ImportServer implements Serializable, Validatable {this.apiToken = apiToken;}- public String getApiEndpoint(String apiPath) {+ private String getApiEndpoint(String apiPath) {return StringUtils.stripEnd(apiUrl, "/") + "/" + StringUtils.stripStart(apiPath, "/");}++ List<String> listProjects() {+ List<String> projects = new ArrayList<>();++ Client client = newClient();+ try {+ String apiEndpoint = getApiEndpoint("/project/search");+ for (JsonNode projectNode: list(client, apiEndpoint, new TaskLogger() {++ @Override+ public void log(String message, String sessionId) {+ logger.info(message);+ }++ })) {+ projects.add(projectNode.get("name").asText());+ }+ } finally {+ client.close();+ }++ Collections.sort(projects);+ return projects;+ }++ private Client newClient() {+ Client client = ClientBuilder.newClient();+ client.property(ClientProperties.FOLLOW_REDIRECTS, true);+ client.register(HttpAuthenticationFeature.basic(getAccountEmail(), getApiToken()));+ return client;+ }++ private GlobalIssueSetting getIssueSetting() {+ return OneDev.getInstance(SettingManager.class).getIssueSetting();+ }++ @Nullable+ private User getUser(Map<String, Optional<User>> users, JsonNode userNode, TaskLogger logger) {+ String accountId = userNode.get("accountId").asText();+ Optional<User> userOpt = users.get(accountId);+ if (userOpt == null) {+ String displayName = null;+ if (userNode.hasNonNull("displayName"))+ displayName = userNode.get("displayName").asText(null);+ if (displayName != null)+ userOpt = Optional.ofNullable(OneDev.getInstance(UserManager.class).findByFullName(displayName));+ else+ userOpt = Optional.empty();+ users.put(accountId, userOpt);+ }+ return userOpt.orElse(null);+ }++ private String getMarkdown(JsonNode contentNode, boolean escape) {+ StringBuilder markdown = new StringBuilder();+ for (JsonNode itemNode: contentNode) {+ String type = itemNode.get("type").asText();+ switch (type) {+ case "text":+ String text = itemNode.get("text").asText();+ if (escape)+ text = HtmlEscape.escapeHtml5(text);++ JsonNode marksNode = itemNode.get("marks");+ if (marksNode != null) {+ for (JsonNode markNode: marksNode) {+ String textType = markNode.get("type").asText(null);+ switch (textType) {+ case "strong":+ text = "**" + text.replace("*", "\\*") + "**";+ break;+ case "em":+ text = "_" + text.replace("_", "\\_") + "_";+ break;+ case "code":+ text = "`" + text + "`";+ break;+ case "textColor":+ text = String.format("<span style='color:%s'>%s</span>",+ markNode.get("attrs").get("color").asText(), text);+ break;+ case "link":+ text = String.format("[%s](%s)", text, markNode.get("attrs").get("href").asText());+ break;+ }+ }+ }+ markdown.append(text);+ break;+ case "mention":+ markdown.append("@" + itemNode.get("attrs").get("text").asText());+ break;+ case "heading":+ markdown.append(StringUtils.repeat('#', itemNode.get("attrs").get("level").asInt())).append(" ");+ markdown.append(getMarkdown(itemNode.get("content"), true)).append("\n\n");+ break;+ case "paragraph":+ markdown.append(getMarkdown(itemNode.get("content"), true)).append("\n\n");+ break;+ case "bulletList":+ for (JsonNode listItemNode: itemNode.get("content"))+ markdown.append("- ").append(getMarkdown(listItemNode.get("content"), true));+ break;+ case "orderedList":+ for (JsonNode listItemNode: itemNode.get("content"))+ markdown.append("1. ").append(getMarkdown(listItemNode.get("content"), true));+ break;+ case "inlineCard":+ markdown.append(" " + StringUtils.substringAfterLast(itemNode.get("attrs").get("url").asText(), "/") + " ");+ break;+ case "emoji":+ markdown.append(" " + itemNode.get("attrs").get("shortName").asText() + " ");+ break;+ case "codeBlock":+ markdown.append("```\n" + getMarkdown(itemNode.get("content"), false) + "\n```\n\n");+ break;+ case "panel":+ String panelType = itemNode.get("attrs").get("panelType").asText();+ switch (panelType) {+ case "info":+ case "note":+ markdown.append(":information_source:\n");+ break;+ case "success":+ markdown.append(":white_check_mark:\n");+ break;+ case "warning":+ markdown.append(":warning:\n");+ break;+ case "error":+ markdown.append(":no_entry:\n");+ break;+ }+ markdown.append(getMarkdown(itemNode.get("content"), true));+ break;+ case "blockquote":+ List<String> lines = Splitter.on('\n').splitToList(getMarkdown(itemNode.get("content"), true));+ for (String line: lines)+ markdown.append("> ").append(line).append("\n");+ break;+ case "rule":+ markdown.append("---\n");+ break;+ case "date":+ markdown.append(DateUtils.formatDate(new Date(itemNode.get("attrs").get("timestamp").asLong())));+ break;+ case "status":+ JsonNode attrsNode = itemNode.get("attrs");+ text = HtmlEscape.escapeHtml5(attrsNode.get("text").asText());+ String color = attrsNode.get("color").asText();+ switch (color) {+ case "yellow":+ markdown.append(String.format("<span style='background:#FFF0B3; padding:2px 4px; border-radius:3px; font-weight:bold; color:#172B4D;'>%s</span>", text));+ break;+ case "neutral":+ markdown.append(String.format("<span style='background:#DFE1E6; padding:2px 4px; border-radius:3px; font-weight:bold; color:#42526E;'>%s</span>", text));+ break;+ case "purple":+ markdown.append(String.format("<span style='background:#EAE6FF; padding:2px 4px; border-radius:3px; font-weight:bold; color:#574BA3;'>%s</span>", text));+ break;+ case "blue":+ markdown.append(String.format("<span style='background:#DEEBFF; padding:2px 4px; border-radius:3px; font-weight:bold; color:#0747A6;'>%s</span>", text));+ break;+ case "red":+ markdown.append(String.format("<span style='background:#FFEBE6; padding:2px 4px; border-radius:3px; font-weight:bold; color:#BF2600;'>%s</span>", text));+ break;+ case "green":+ markdown.append(String.format("<span style='background:#E3FCEF; padding:2px 4px; border-radius:3px; font-weight:bold; color:#006644;'>%s</span>", text));+ break;+ default:+ markdown.append(text);+ }+ break;+ case "table":+ List<List<String>> tableData = new ArrayList<>();+ for (JsonNode rowNode: itemNode.get("content")) {+ List<String> columns = new ArrayList<>();+ for (JsonNode columnNode: rowNode.get("content"))+ columns.add(getMarkdown(columnNode.get("content"), true).replace("\n", "<br>"));+ tableData.add(columns);+ }+ if (!tableData.isEmpty()) {+ if (!tableData.get(0).isEmpty()) {+ markdown.append("|");+ for (int i=0; i<tableData.get(0).size(); i++)+ markdown.append("-----|");+ markdown.append("\n");+ for (List<String> row: tableData) {+ markdown.append("|");+ for (String column: row)+ markdown.append(" ").append(column).append(" |");+ markdown.append("\n");+ }+ markdown.append("\n");+ }+ }+ break;+ }+ }+ return markdown.toString();+ }++ String importProjects(ImportProjects projects, ImportOption option, boolean dryRun, TaskLogger logger) {+ Collection<Long> projectIds = new ArrayList<>();++ Map<String, JsonNode> projectNodes = getProjectNodes(logger);++ Client client = newClient();+ try {+ Map<String, Optional<User>> users = new HashMap<>();+ ImportResult result = new ImportResult();++ for (ProjectMapping projectMapping: projects.getProjectMappings()) {+ String jiraProject = projectMapping.getJiraProject();+ JsonNode projectNode = projectNodes.get(jiraProject);+ if (projectNode == null)+ throw new ExplicitException("Unable to find project: " + jiraProject);++ String apiEndpoint = getApiEndpoint("/project/" + projectNode.get("id").asText());++ // Get more detail project information+ projectNode = JerseyUtils.get(client, apiEndpoint, logger);++ ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);+ Project project = projectManager.initialize(projectMapping.getOneDevProject());+ Preconditions.checkState(project.isNew());++ project.setDescription(projectNode.get("description").asText(null));++ if (!dryRun) {+ OneDev.getInstance(ProjectManager.class).create(project);+ projectIds.add(project.getId());+ }++ logger.log("Importing issues from project " + jiraProject + "...");+ ImportResult currentResult = importIssues(projectNode, project, true, option, users, dryRun, logger);+ result.nonExistentLogins.addAll(currentResult.nonExistentLogins);+ result.errorAttachments.addAll(currentResult.errorAttachments);+ result.tooLargeAttachments.addAll(currentResult.tooLargeAttachments);+ result.unmappedIssuePriorities.addAll(currentResult.unmappedIssuePriorities);+ result.unmappedIssueStatuses.addAll(currentResult.unmappedIssueStatuses);+ result.unmappedIssueTypes.addAll(currentResult.unmappedIssueTypes);+ }++ return result.toHtml("Projects imported successfully");+ } catch (Exception e) {+ for (Long projectId: projectIds)+ OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);+ throw new RuntimeException(e);+ } finally {+ client.close();+ }+ }++ String importIssues(Project project, String jiraProject, ImportOption option,+ boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {+ Map<String, JsonNode> projectNodes = getProjectNodes(logger);++ Client client = newClient();+ try {+ Map<String, Optional<User>> users = new HashMap<>();+ JsonNode projectNode = projectNodes.get(jiraProject);+ if (projectNode == null)+ throw new ExplicitException("Unable to find project: " + jiraProject);+ ImportResult result = importIssues(projectNode, project, retainIssueNumbers,+ option, users, dryRun, logger);+ return result.toHtml("Issues imported successfully");+ } finally {+ client.close();+ }+ }++ private ImportResult importIssues(JsonNode jiraProject, Project oneDevProject,+ boolean retainIssueNumbers, ImportOption option,+ Map<String, Optional<User>> users, boolean dryRun, TaskLogger logger) {+ Client client = newClient();+ try {+ Set<String> unmappedIssueStatuses = new HashSet<>();+ Set<String> unmappedIssueTypes = new HashSet<>();+ Set<String> unmappedIssuePriorities = new HashSet<>();+ Set<String> nonExistentLogins = new HashSet<>();+ Set<String> tooLargeAttachments = new LinkedHashSet<>();+ Set<String> errorAttachments = new HashSet<>();++ Map<String, Pair<FieldSpec, String>> typeMappings = new HashMap<>();++ for (IssueTypeMapping mapping: option.getIssueTypeMappings()) {+ String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");+ String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");+ FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);+ if (fieldSpec == null)+ throw new ExplicitException("No field spec found: " + oneDevFieldName);+ typeMappings.put(mapping.getJiraIssueType(), new Pair<>(fieldSpec, oneDevFieldValue));+ }++ Map<String, Pair<FieldSpec, String>> priorityMappings = new HashMap<>();++ for (IssuePriorityMapping mapping: option.getIssuePriorityMappings()) {+ String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");+ String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");+ FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);+ if (fieldSpec == null)+ throw new ExplicitException("No field spec found: " + oneDevFieldName);+ priorityMappings.put(mapping.getJiraIssuePriority(), new Pair<>(fieldSpec, oneDevFieldValue));+ }++ String initialIssueState = getIssueSetting().getInitialStateSpec().getName();++ List<Issue> issues = new ArrayList<>();++ Map<Long, Long> issueNumberMappings = new HashMap<>();++ Map<String, String> statusMappings = new HashMap<>();+ for (IssueStatusMapping mapping: option.getIssueStatusMappings())+ statusMappings.put(mapping.getJiraIssueStatus(), mapping.getOneDevIssueState());++ Map<String, String> untranslatedStatusNames = new HashMap<>();+ String apiEndpoint = getApiEndpoint("/status");+ for (JsonNode statusNode: list(client, apiEndpoint, logger)) {+ String untranslated = statusNode.get("untranslatedName").asText();+ untranslatedStatusNames.put(statusNode.get("id").asText(), untranslated);+ }++ Map<String, String> untranslatedTypeNames = new HashMap<>();+ apiEndpoint = getApiEndpoint("/issuetype/project?projectId=" + jiraProject.get("id").asText());+ for (JsonNode typeNode: list(client, apiEndpoint, logger))+ untranslatedTypeNames.put(typeNode.get("id").asText(), typeNode.get("untranslatedName").asText());++ AtomicInteger numOfImportedIssues = new AtomicInteger(0);+ PageDataConsumer pageDataConsumer = new PageDataConsumer() {++ private String joinAsMultilineHtml(List<String> values) {+ List<String> escapedValues = new ArrayList<>();+ for (String value: values)+ escapedValues.add(HtmlEscape.escapeHtml5(value));+ return StringUtils.join(escapedValues, "<br>");+ }++ @Override+ public void consume(List<JsonNode> pageData) throws InterruptedException {+ for (JsonNode issueNode: pageData) {+ if (Thread.interrupted())+ throw new InterruptedException();++ String issueKey = issueNode.get("key").asText();+ String apiEndpoint = getApiEndpoint("/issue/" + issueKey);+ issueNode = JerseyUtils.get(client, apiEndpoint, logger);++ Map<String, String> extraIssueInfo = new LinkedHashMap<>();++ Issue issue = new Issue();+ issue.setProject(oneDevProject);++ issue.setNumberScope(oneDevProject.getForkRoot());++ Long oldNumber = Long.valueOf(StringUtils.substringAfterLast(issueNode.get("key").asText(), "-"));+ Long newNumber;+ if (dryRun || retainIssueNumbers)+ newNumber = oldNumber;+ else+ newNumber = OneDev.getInstance(IssueManager.class).getNextNumber(oneDevProject);+ issue.setNumber(newNumber);+ issueNumberMappings.put(oldNumber, newNumber);++ JsonNode fieldsNode = issueNode.get("fields");++ issue.setTitle(fieldsNode.get("summary").asText());+ String description = "";+ JsonNode descriptionNode = fieldsNode.get("description");+ if (descriptionNode != null && descriptionNode.hasNonNull("content")) {+ description = getMarkdown(descriptionNode.get("content"), true);+ if (StringUtils.isNotBlank("description"))+ description = "#### Description\n\n" + description;+ }++ JsonNode environmentNode = fieldsNode.get("environment");+ if (environmentNode != null && environmentNode.hasNonNull("content")) {+ String environment = getMarkdown(environmentNode.get("content"), true);+ if (StringUtils.isNotBlank(environment)) {+ if (StringUtils.isNotBlank(description))+ description += "\n";+ description += "#### Environment\n\n" + environment;+ }+ }++ if (StringUtils.isNotBlank(description))+ issue.setDescription(description);++ String statusId = fieldsNode.get("status").get("id").asText();+ String untranslatedStatusName = untranslatedStatusNames.get(statusId);+ if (untranslatedStatusName != null) {+ String statusName = statusMappings.get(untranslatedStatusName);+ if (statusName != null) {+ issue.setState(statusName);+ } else {+ unmappedIssueStatuses.add(untranslatedStatusName);+ issue.setState(initialIssueState);+ }+ } else {+ throw new ExplicitException("Can not find JIRA status of id: " + statusId);+ }++ String typeId = fieldsNode.get("issuetype").get("id").asText();+ String untranslatedTypeName = untranslatedTypeNames.get(typeId);+ if (untranslatedTypeName != null) {+ Pair<FieldSpec, String> mapped = typeMappings.get(untranslatedTypeName);+ if (mapped != null) {+ IssueField typeField = new IssueField();+ typeField.setIssue(issue);+ typeField.setName(mapped.getFirst().getName());+ typeField.setType(InputSpec.ENUMERATION);+ typeField.setValue(mapped.getSecond());+ typeField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));+ issue.getFields().add(typeField);+ } else {+ extraIssueInfo.put("Type", HtmlEscape.escapeHtml5(untranslatedTypeName));+ unmappedIssueTypes.add(untranslatedTypeName);+ }+ } else {+ String errorMessage = String.format("Can not find JIRA issue type by id (project: %s, issue type id: %s)",+ jiraProject.get("name").asText(), typeId);+ throw new ExplicitException(errorMessage);+ }++ String priorityName = fieldsNode.get("priority").get("name").asText();+ Pair<FieldSpec, String> mapped = priorityMappings.get(priorityName);+ if (mapped != null) {+ IssueField priorityField = new IssueField();+ priorityField.setIssue(issue);+ priorityField.setName(mapped.getFirst().getName());+ priorityField.setType(InputSpec.ENUMERATION);+ priorityField.setValue(mapped.getSecond());+ priorityField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));+ issue.getFields().add(priorityField);+ } else {+ extraIssueInfo.put("Priority", HtmlEscape.escapeHtml5(priorityName));+ unmappedIssuePriorities.add(priorityName);+ }++ List<String> components = new ArrayList<>();+ for (JsonNode componentNode: fieldsNode.get("components"))+ components.add(componentNode.get("name").asText());+ if (!components.isEmpty())+ extraIssueInfo.put("Components", joinAsMultilineHtml(components));++ List<String> affectedVersions = new ArrayList<>();+ for (JsonNode versionNode: fieldsNode.get("versions"))+ affectedVersions.add(versionNode.get("name").asText());+ if (!affectedVersions.isEmpty())+ extraIssueInfo.put("Affected Versions", joinAsMultilineHtml(affectedVersions));++ List<String> fixVersions = new ArrayList<>();+ for (JsonNode versionNode: fieldsNode.get("fixVersions"))+ fixVersions.add(versionNode.get("name").asText());+ if (!fixVersions.isEmpty())+ extraIssueInfo.put("Fix Versions", joinAsMultilineHtml(fixVersions));++ if (fieldsNode.hasNonNull("labels")) {+ List<String> labels = new ArrayList<>();+ for (JsonNode labelNode: fieldsNode.get("labels"))+ labels.add(labelNode.asText());+ if (!labels.isEmpty())+ extraIssueInfo.put("Labels", joinAsMultilineHtml(labels));+ }++ JsonNode authorNode = fieldsNode.get("reporter");+ if (authorNode == null)+ authorNode = fieldsNode.get("creator");+ if (authorNode != null) {+ User user = getUser(users, authorNode, logger);+ if (user != null) {+ issue.setSubmitter(user);+ } else {+ issue.setSubmitter(OneDev.getInstance(UserManager.class).getUnknown());+ nonExistentLogins.add(authorNode.get("displayName").asText());+ }+ } else {+ issue.setSubmitter(OneDev.getInstance(UserManager.class).getUnknown());+ }++ if (fieldsNode.hasNonNull("assignee")) {+ JsonNode assigneeNode = fieldsNode.get("assignee");+ IssueField assigneeField = new IssueField();+ assigneeField.setIssue(issue);+ assigneeField.setName(option.getAssigneeIssueField());+ assigneeField.setType(InputSpec.USER);++ User user = getUser(users, assigneeNode, logger);+ if (user != null) {+ assigneeField.setValue(user.getName());+ issue.getFields().add(assigneeField);+ } else {+ nonExistentLogins.add(assigneeNode.get("displayName").asText());+ }+ }++ issue.setSubmitDate(ISODateTimeFormat.dateTime()+ .parseDateTime(fieldsNode.get("created").asText())+ .toDate());++ LastUpdate lastUpdate = new LastUpdate();+ lastUpdate.setActivity("Opened");+ lastUpdate.setDate(issue.getSubmitDate());+ lastUpdate.setUser(issue.getSubmitter());+ issue.setLastUpdate(lastUpdate);++ if (option.getDueDateIssueField() != null) {+ String dueDate = fieldsNode.get("duedate").asText(null);+ if (dueDate != null) {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(option.getDueDateIssueField());+ issueField.setType(InputSpec.DATE);+ issueField.setValue(dueDate);+ issue.getFields().add(issueField);+ }+ }++ if (option.getTimeEstimateIssueField() != null) {+ String timeEstimate = fieldsNode.get("timeestimate").asText(null);+ if (timeEstimate != null) {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(option.getTimeEstimateIssueField());+ issueField.setType(InputSpec.WORKING_PERIOD);+ issueField.setValue(DateUtils.formatWorkingPeriod(Integer.valueOf(timeEstimate)/60));+ issue.getFields().add(issueField);+ }+ }+ if (option.getTimeSpentIssueField() != null) {+ String timeSpent = fieldsNode.get("timespent").asText(null);+ if (timeSpent != null) {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(option.getTimeSpentIssueField());+ issueField.setType(InputSpec.WORKING_PERIOD);+ issueField.setValue(DateUtils.formatWorkingPeriod(Integer.valueOf(timeSpent)/60));+ issue.getFields().add(issueField);+ }+ }++ if (fieldsNode.hasNonNull("subtasks")) {+ List<String> subtasks = new ArrayList<>();+ for (JsonNode subtaskNode: fieldsNode.get("subtasks"))+ subtasks.add(subtaskNode.get("key").asText());+ if (!subtasks.isEmpty())+ extraIssueInfo.put("Sub Tasks", StringUtils.join(subtasks, "<br>"));+ }++ if (fieldsNode.hasNonNull("parent"))+ extraIssueInfo.put("Parent Task", fieldsNode.get("parent").get("key").asText());++ if (fieldsNode.hasNonNull("issuelinks")) {+ Map<String, List<String>> issueLinks = new LinkedHashMap<>();+ for (JsonNode issueLinkNode: fieldsNode.get("issuelinks")) {+ if (issueLinkNode.get("inwardIssue") != null) {+ String type = issueLinkNode.get("type").get("inward").asText();+ List<String> linkedIssues = issueLinks.get(type);+ if (linkedIssues == null) {+ linkedIssues = new ArrayList<>();+ issueLinks.put(type, linkedIssues);+ }+ linkedIssues.add(issueLinkNode.get("inwardIssue").get("key").asText());+ } else {+ String type = issueLinkNode.get("type").get("outward").asText();+ List<String> linkedIssues = issueLinks.get(type);+ if (linkedIssues == null) {+ linkedIssues = new ArrayList<>();+ issueLinks.put(type, linkedIssues);+ }+ linkedIssues.add(issueLinkNode.get("outwardIssue").get("key").asText());+ }+ }+ for (Map.Entry<String, List<String>> entry: issueLinks.entrySet())+ extraIssueInfo.put(entry.getKey(), StringUtils.join(entry.getValue(), "<br>"));+ }++ if (fieldsNode.hasNonNull("comment") && fieldsNode.get("comment").hasNonNull("comments")) {+ for (JsonNode commentNode: fieldsNode.get("comment").get("comments")) {+ String commentContent = getMarkdown(commentNode.get("body").get("content"), true);+ if (StringUtils.isNotBlank(commentContent)) {+ IssueComment comment = new IssueComment();+ comment.setIssue(issue);+ comment.setContent(commentContent);++ comment.setDate(ISODateTimeFormat.dateTime()+ .parseDateTime(commentNode.get("created").asText())+ .toDate());++ authorNode = commentNode.get("author");+ if (authorNode != null) {+ User user = getUser(users, authorNode, logger);+ if (user != null) {+ comment.setUser(user);+ } else {+ comment.setUser(OneDev.getInstance(UserManager.class).getUnknown());+ nonExistentLogins.add(authorNode.get("displayName").asText());+ }+ } else {+ comment.setUser(OneDev.getInstance(UserManager.class).getUnknown());+ }++ issue.getComments().add(comment);+ }+ }+ }++ issue.setCommentCount(issue.getComments().size());++ Set<String> fieldAndValues = new HashSet<>();+ for (IssueField field: issue.getFields()) {+ String fieldAndValue = field.getName() + "::" + field.getValue();+ if (!fieldAndValues.add(fieldAndValue)) {+ String errorMessage = String.format(+ "Duplicate issue field mapping (issue: %s, field: %s)",+ issueKey, fieldAndValue);+ throw new ExplicitException(errorMessage);+ }+ }++ if (!extraIssueInfo.isEmpty()) {+ StringBuilder builder = new StringBuilder("|");+ for (String key: extraIssueInfo.keySet())+ builder.append(key).append("|");+ builder.append("\n|");+ extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));+ builder.append("\n|");+ for (String value: extraIssueInfo.values())+ builder.append(value).append("|");++ if (issue.getDescription() != null)+ issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());+ else+ issue.setDescription(builder.toString());+ }++ if (!dryRun) {+ List<String> attachments = new ArrayList<>();++ if (fieldsNode.hasNonNull("attachment")) {+ long maxUploadFileSize = OneDev.getInstance(SettingManager.class)+ .getPerformanceSetting().getMaxUploadFileSize()*1L*1024*1024;+ for (JsonNode attachmentNode: fieldsNode.get("attachment")) {+ String attachmentName = attachmentNode.get("filename").asText();+ int attachmentSize = attachmentNode.get("size").asInt();+ if (attachmentSize > maxUploadFileSize) {+ tooLargeAttachments.add(issueKey + ":" + attachmentName);+ } else {+ String endpoint = attachmentNode.get("content").asText();+ WebTarget target = client.target(endpoint);+ Invocation.Builder builder = target.request();+ try (Response response = builder.get()) {+ String errorMessage = JerseyUtils.checkStatus(endpoint, response);+ if (errorMessage != null) {+ throw new ExplicitException(String.format(+ "Error downloading attachment (url: %s, error message: %s)",+ endpoint, errorMessage));+ }+ try (InputStream is = response.readEntity(InputStream.class)) {+ String oneDevAttachmentName = oneDevProject.saveAttachment(issue.getUUID(), attachmentName, is);+ String oneDevAttachmentUrl = oneDevProject.getAttachmentUrlPath(issue.getUUID(), oneDevAttachmentName);+ attachments.add("[" + oneDevAttachmentName + "](" + oneDevAttachmentUrl + ")");+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ }+ }+ }+ }++ if (!attachments.isEmpty()) {+ description = issue.getDescription();+ if (StringUtils.isNotBlank(description))+ description += "\n";+ description += "#### Attachments\n" + Joiner.on(" • ").join(attachments);+ issue.setDescription(description);+ }+ }++ issues.add(issue);+ }+ logger.log("Imported " + numOfImportedIssues.addAndGet(pageData.size()) + " issues");+ }++ };++ String jiraProjectName = jiraProject.get("name").asText();+ String jql = String.format("project = \"%s\" order by created DESC", jiraProjectName);+ jql = URLEncoder.encode(jql, StandardCharsets.UTF_8.name());+ apiEndpoint = getApiEndpoint("/search?jql=" + jql);+ list(client, apiEndpoint, pageDataConsumer, logger);++ if (!dryRun) {+ ReferenceMigrator migrator = new ReferenceMigrator(Issue.class, issueNumberMappings);+ String jiraProjectKey = jiraProject.get("key").asText();+ Dao dao = OneDev.getInstance(Dao.class);+ for (Issue issue: issues) {+ if (issue.getDescription() != null)+ issue.setDescription(migrator.migratePrefixed(issue.getDescription(), jiraProjectKey + "-"));++ OneDev.getInstance(IssueManager.class).save(issue);+ for (IssueSchedule schedule: issue.getSchedules())+ dao.persist(schedule);+ for (IssueField field: issue.getFields())+ dao.persist(field);+ for (IssueComment comment: issue.getComments()) {+ comment.setContent(migrator.migratePrefixed(comment.getContent(), jiraProjectKey + "-"));+ dao.persist(comment);+ }+ }+ }++ ImportResult result = new ImportResult();+ result.unmappedIssueStatuses.addAll(unmappedIssueStatuses);+ result.unmappedIssueTypes.addAll(unmappedIssueTypes);+ result.unmappedIssuePriorities.addAll(unmappedIssuePriorities);+ result.nonExistentLogins.addAll(nonExistentLogins);+ result.tooLargeAttachments.addAll(tooLargeAttachments);+ result.errorAttachments.addAll(errorAttachments);++ return result;+ } catch (UnsupportedEncodingException e) {+ throw new RuntimeException(e);+ } finally {+ client.close();+ }+ }++ ImportOption buildImportOption(Collection<String> projectNames) {+ ImportOption option = new ImportOption();++ TaskLogger taskLogger = new TaskLogger() {++ @Override+ public void log(String message, String sessionId) {+ logger.info(message);+ }++ };+ Map<String, JsonNode> projectNodes = getProjectNodes(taskLogger);++ Client client = newClient();+ try {+ String apiEndpoint = getApiEndpoint("/status");+ List<JsonNode> statusNodes = list(client, apiEndpoint, taskLogger);++ Set<String> issueTypes = new LinkedHashSet<>();+ Set<String> issueStatuses = new LinkedHashSet<>();+ for (String projectName: projectNames) {+ JsonNode projectNode = projectNodes.get(projectName);+ if (projectNode == null)+ throw new ExplicitException("Unable to find project: " + projectName);+ apiEndpoint = getApiEndpoint("/issuetype/project?projectId=" + projectNode.get("id").asText());+ for (JsonNode issueTypeNode: list(client, apiEndpoint, taskLogger))+ issueTypes.add(issueTypeNode.get("untranslatedName").asText());+ if (projectNode.get("style").asText().equals("next-gen")) {+ for (JsonNode statusNode: statusNodes) {+ JsonNode scopeNode = statusNode.get("scope");+ if (scopeNode != null && scopeNode.get("project").get("id").asText().equals(projectNode.get("id").asText()))+ issueStatuses.add(statusNode.get("untranslatedName").asText());+ }+ } else {+ for (JsonNode statusNode: statusNodes) {+ if (statusNode.get("scope") == null)+ issueStatuses.add(statusNode.get("untranslatedName").asText());+ }+ }+ }+ for (String issueType: issueTypes) {+ IssueTypeMapping mapping = new IssueTypeMapping();+ mapping.setJiraIssueType(issueType);+ option.getIssueTypeMappings().add(mapping);+ }+ for (String issueStatus: issueStatuses) {+ IssueStatusMapping mapping = new IssueStatusMapping();+ mapping.setJiraIssueStatus(issueStatus);+ option.getIssueStatusMappings().add(mapping);+ }+ apiEndpoint = getApiEndpoint("/priority");+ for (JsonNode priorityNode: list(client, apiEndpoint, taskLogger)) {+ IssuePriorityMapping mapping = new IssuePriorityMapping();+ mapping.setJiraIssuePriority(priorityNode.get("name").asText());+ option.getIssuePriorityMappings().add(mapping);+ }+ } finally {+ client.close();+ }+ return option;+ }++ private Map<String, JsonNode> getProjectNodes(TaskLogger logger) {+ Map<String, JsonNode> projectNodes = new LinkedHashMap<>();++ Client client = newClient();+ try {+ for (JsonNode projectNode: list(client, getApiEndpoint("/project/search"), logger))+ projectNodes.put(projectNode.get("name").asText(), projectNode);+ } finally {+ client.close();+ }+ return projectNodes;+ }++ private List<JsonNode> list(Client client, String apiEndpoint, TaskLogger logger) {+ List<JsonNode> result = new ArrayList<>();+ list(client, apiEndpoint, new PageDataConsumer() {++ @Override+ public void consume(List<JsonNode> pageData) {+ result.addAll(pageData);+ }++ }, logger);+ return result;+ }++ private void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer,+ TaskLogger logger) {+ URI uri;+ try {+ uri = new URIBuilder(apiEndpoint)+ .addParameter("maxResults", String.valueOf(PER_PAGE)).build();+ } catch (URISyntaxException e) {+ throw new RuntimeException(e);+ }++ int page = 0;+ while (true) {+ try {+ URIBuilder builder = new URIBuilder(uri);+ builder.addParameter("startAt", String.valueOf(page*PER_PAGE));+ List<JsonNode> pageData = new ArrayList<>();++ JsonNode resultNode = JerseyUtils.get(client, builder.build().toString(), logger);+ if (resultNode.isArray()) {+ for (JsonNode each: resultNode)+ pageData.add(each);+ } else {+ if (resultNode.hasNonNull("values")) {+ for (JsonNode each: resultNode.get("values"))+ pageData.add(each);+ } else if (resultNode.hasNonNull("issues")) {+ for (JsonNode each: resultNode.get("issues"))+ pageData.add(each);+ } else if (resultNode.hasNonNull("comments")) {+ for (JsonNode each: resultNode.get("comments"))+ pageData.add(each);+ }+ }+ pageDataConsumer.consume(pageData);+ if (pageData.size() < PER_PAGE)+ break;+ page++;+ } catch (URISyntaxException | InterruptedException e) {+ throw new RuntimeException(e);+ }+ }+ }+@Overridepublic boolean isValid(ConstraintValidatorContext context) {Client client = ClientBuilder.newClient();@@ -122,11 +1032,4 @@ public class ImportServer implements Serializable, Validatable {return true;}- Client newClient() {- Client client = ClientBuilder.newClient();- client.property(ClientProperties.FOLLOW_REDIRECTS, true);- client.register(HttpAuthenticationFeature.basic(getAccountEmail(), getApiToken()));- return client;- }-}diff --git a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportUtils.java b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportUtils.javadeleted file mode 100644index e920517806..0000000000--- a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportUtils.java+++ /dev/null@@ -1,817 +0,0 @@-package io.onedev.server.plugin.imports.jiracloud;--import java.io.IOException;-import java.io.InputStream;-import java.io.UnsupportedEncodingException;-import java.net.URI;-import java.net.URISyntaxException;-import java.net.URLEncoder;-import java.nio.charset.StandardCharsets;-import java.util.ArrayList;-import java.util.Collection;-import java.util.Date;-import java.util.HashMap;-import java.util.HashSet;-import java.util.LinkedHashMap;-import java.util.LinkedHashSet;-import java.util.List;-import java.util.Map;-import java.util.Optional;-import java.util.Set;-import java.util.concurrent.atomic.AtomicInteger;--import javax.annotation.Nullable;-import javax.ws.rs.client.Client;-import javax.ws.rs.client.Invocation;-import javax.ws.rs.client.WebTarget;-import javax.ws.rs.core.Response;--import org.apache.commons.lang3.StringUtils;-import org.apache.http.client.utils.URIBuilder;-import org.joda.time.format.ISODateTimeFormat;-import org.unbescape.html.HtmlEscape;--import com.fasterxml.jackson.databind.JsonNode;-import com.google.common.base.Joiner;-import com.google.common.base.Splitter;--import io.onedev.commons.utils.ExplicitException;-import io.onedev.commons.utils.TaskLogger;-import io.onedev.server.OneDev;-import io.onedev.server.entitymanager.IssueManager;-import io.onedev.server.entitymanager.SettingManager;-import io.onedev.server.entitymanager.UserManager;-import io.onedev.server.entityreference.ReferenceMigrator;-import io.onedev.server.model.Issue;-import io.onedev.server.model.IssueComment;-import io.onedev.server.model.IssueField;-import io.onedev.server.model.IssueSchedule;-import io.onedev.server.model.Project;-import io.onedev.server.model.User;-import io.onedev.server.model.support.LastUpdate;-import io.onedev.server.model.support.administration.GlobalIssueSetting;-import io.onedev.server.model.support.inputspec.InputSpec;-import io.onedev.server.model.support.issue.field.spec.FieldSpec;-import io.onedev.server.persistence.dao.Dao;-import io.onedev.server.util.DateUtils;-import io.onedev.server.util.JerseyUtils;-import io.onedev.server.util.JerseyUtils.PageDataConsumer;-import io.onedev.server.util.Pair;--public class ImportUtils {-- static final String NAME = "JIRA Cloud";-- static final int PER_PAGE = 50;-- static GlobalIssueSetting getIssueSetting() {- return OneDev.getInstance(SettingManager.class).getIssueSetting();- }-- @Nullable- static User getUser(Map<String, Optional<User>> users, JsonNode userNode, TaskLogger logger) {- String accountId = userNode.get("accountId").asText();- Optional<User> userOpt = users.get(accountId);- if (userOpt == null) {- String displayName = null;- if (userNode.hasNonNull("displayName"))- displayName = userNode.get("displayName").asText(null);- if (displayName != null)- userOpt = Optional.ofNullable(OneDev.getInstance(UserManager.class).findByFullName(displayName));- else- userOpt = Optional.empty();- users.put(accountId, userOpt);- }- return userOpt.orElse(null);- }-- private static String getMarkdown(JsonNode contentNode, boolean escape) {- StringBuilder markdown = new StringBuilder();- for (JsonNode itemNode: contentNode) {- String type = itemNode.get("type").asText();- switch (type) {- case "text":- String text = itemNode.get("text").asText();- if (escape)- text = HtmlEscape.escapeHtml5(text);-- JsonNode marksNode = itemNode.get("marks");- if (marksNode != null) {- for (JsonNode markNode: marksNode) {- String textType = markNode.get("type").asText(null);- switch (textType) {- case "strong":- text = "**" + text.replace("*", "\\*") + "**";- break;- case "em":- text = "_" + text.replace("_", "\\_") + "_";- break;- case "code":- text = "`" + text + "`";- break;- case "textColor":- text = String.format("<span style='color:%s'>%s</span>",- markNode.get("attrs").get("color").asText(), text);- break;- case "link":- text = String.format("[%s](%s)", text, markNode.get("attrs").get("href").asText());- break;- }- }- }- markdown.append(text);- break;- case "mention":- markdown.append("@" + itemNode.get("attrs").get("text").asText());- break;- case "heading":- markdown.append(StringUtils.repeat('#', itemNode.get("attrs").get("level").asInt())).append(" ");- markdown.append(getMarkdown(itemNode.get("content"), true)).append("\n\n");- break;- case "paragraph":- markdown.append(getMarkdown(itemNode.get("content"), true)).append("\n\n");- break;- case "bulletList":- for (JsonNode listItemNode: itemNode.get("content"))- markdown.append("- ").append(getMarkdown(listItemNode.get("content"), true));- break;- case "orderedList":- for (JsonNode listItemNode: itemNode.get("content"))- markdown.append("1. ").append(getMarkdown(listItemNode.get("content"), true));- break;- case "inlineCard":- markdown.append(" " + StringUtils.substringAfterLast(itemNode.get("attrs").get("url").asText(), "/") + " ");- break;- case "emoji":- markdown.append(" " + itemNode.get("attrs").get("shortName").asText() + " ");- break;- case "codeBlock":- markdown.append("```\n" + getMarkdown(itemNode.get("content"), false) + "\n```\n\n");- break;- case "panel":- String panelType = itemNode.get("attrs").get("panelType").asText();- switch (panelType) {- case "info":- case "note":- markdown.append(":information_source:\n");- break;- case "success":- markdown.append(":white_check_mark:\n");- break;- case "warning":- markdown.append(":warning:\n");- break;- case "error":- markdown.append(":no_entry:\n");- break;- }- markdown.append(getMarkdown(itemNode.get("content"), true));- break;- case "blockquote":- List<String> lines = Splitter.on('\n').splitToList(getMarkdown(itemNode.get("content"), true));- for (String line: lines)- markdown.append("> ").append(line).append("\n");- break;- case "rule":- markdown.append("---\n");- break;- case "date":- markdown.append(DateUtils.formatDate(new Date(itemNode.get("attrs").get("timestamp").asLong())));- break;- case "status":- JsonNode attrsNode = itemNode.get("attrs");- text = HtmlEscape.escapeHtml5(attrsNode.get("text").asText());- String color = attrsNode.get("color").asText();- switch (color) {- case "yellow":- markdown.append(String.format("<span style='background:#FFF0B3; padding:2px 4px; border-radius:3px; font-weight:bold; color:#172B4D;'>%s</span>", text));- break;- case "neutral":- markdown.append(String.format("<span style='background:#DFE1E6; padding:2px 4px; border-radius:3px; font-weight:bold; color:#42526E;'>%s</span>", text));- break;- case "purple":- markdown.append(String.format("<span style='background:#EAE6FF; padding:2px 4px; border-radius:3px; font-weight:bold; color:#574BA3;'>%s</span>", text));- break;- case "blue":- markdown.append(String.format("<span style='background:#DEEBFF; padding:2px 4px; border-radius:3px; font-weight:bold; color:#0747A6;'>%s</span>", text));- break;- case "red":- markdown.append(String.format("<span style='background:#FFEBE6; padding:2px 4px; border-radius:3px; font-weight:bold; color:#BF2600;'>%s</span>", text));- break;- case "green":- markdown.append(String.format("<span style='background:#E3FCEF; padding:2px 4px; border-radius:3px; font-weight:bold; color:#006644;'>%s</span>", text));- break;- default:- markdown.append(text);- }- break;- case "table":- List<List<String>> tableData = new ArrayList<>();- for (JsonNode rowNode: itemNode.get("content")) {- List<String> columns = new ArrayList<>();- for (JsonNode columnNode: rowNode.get("content"))- columns.add(getMarkdown(columnNode.get("content"), true).replace("\n", "<br>"));- tableData.add(columns);- }- if (!tableData.isEmpty()) {- if (!tableData.get(0).isEmpty()) {- markdown.append("|");- for (int i=0; i<tableData.get(0).size(); i++)- markdown.append("-----|");- markdown.append("\n");- for (List<String> row: tableData) {- markdown.append("|");- for (String column: row)- markdown.append(" ").append(column).append(" |");- markdown.append("\n");- }- markdown.append("\n");- }- }- break;- }- }- return markdown.toString();- }-- static ImportResult importIssues(ImportServer server, JsonNode jiraProject, Project oneDevProject,- boolean retainIssueNumbers, ImportOption importOption,- Map<String, Optional<User>> users, boolean dryRun, TaskLogger logger) {- Client client = server.newClient();- try {- Set<String> unmappedIssueStatuses = new HashSet<>();- Set<String> unmappedIssueTypes = new HashSet<>();- Set<String> unmappedIssuePriorities = new HashSet<>();- Set<String> nonExistentLogins = new HashSet<>();- Set<String> tooLargeAttachments = new LinkedHashSet<>();- Set<String> errorAttachments = new HashSet<>();-- Map<String, Pair<FieldSpec, String>> typeMappings = new HashMap<>();-- for (IssueTypeMapping mapping: importOption.getIssueTypeMappings()) {- String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");- String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");- FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);- if (fieldSpec == null)- throw new ExplicitException("No field spec found: " + oneDevFieldName);- typeMappings.put(mapping.getJiraIssueType(), new Pair<>(fieldSpec, oneDevFieldValue));- }-- Map<String, Pair<FieldSpec, String>> priorityMappings = new HashMap<>();-- for (IssuePriorityMapping mapping: importOption.getIssuePriorityMappings()) {- String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");- String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");- FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);- if (fieldSpec == null)- throw new ExplicitException("No field spec found: " + oneDevFieldName);- priorityMappings.put(mapping.getJiraIssuePriority(), new Pair<>(fieldSpec, oneDevFieldValue));- }-- String initialIssueState = getIssueSetting().getInitialStateSpec().getName();-- List<Issue> issues = new ArrayList<>();-- Map<Long, Long> issueNumberMappings = new HashMap<>();-- Map<String, String> statusMappings = new HashMap<>();- for (IssueStatusMapping mapping: importOption.getIssueStatusMappings())- statusMappings.put(mapping.getJiraIssueStatus(), mapping.getOneDevIssueState());-- Map<String, String> untranslatedStatusNames = new HashMap<>();- String apiEndpoint = server.getApiEndpoint("/status");- for (JsonNode statusNode: list(client, apiEndpoint, logger)) {- String untranslated = statusNode.get("untranslatedName").asText();- untranslatedStatusNames.put(statusNode.get("id").asText(), untranslated);- }-- Map<String, String> untranslatedTypeNames = new HashMap<>();- apiEndpoint = server.getApiEndpoint("/issuetype/project?projectId=" + jiraProject.get("id").asText());- for (JsonNode typeNode: list(client, apiEndpoint, logger))- untranslatedTypeNames.put(typeNode.get("id").asText(), typeNode.get("untranslatedName").asText());-- AtomicInteger numOfImportedIssues = new AtomicInteger(0);- PageDataConsumer pageDataConsumer = new PageDataConsumer() {-- private String joinAsMultilineHtml(List<String> values) {- List<String> escapedValues = new ArrayList<>();- for (String value: values)- escapedValues.add(HtmlEscape.escapeHtml5(value));- return StringUtils.join(escapedValues, "<br>");- }-- @Override- public void consume(List<JsonNode> pageData) throws InterruptedException {- for (JsonNode issueNode: pageData) {- if (Thread.interrupted())- throw new InterruptedException();-- String issueKey = issueNode.get("key").asText();- String apiEndpoint = server.getApiEndpoint("/issue/" + issueKey);- issueNode = JerseyUtils.get(client, apiEndpoint, logger);-- Map<String, String> extraIssueInfo = new LinkedHashMap<>();-- Issue issue = new Issue();- issue.setProject(oneDevProject);-- issue.setNumberScope(oneDevProject.getForkRoot());-- Long oldNumber = Long.valueOf(StringUtils.substringAfterLast(issueNode.get("key").asText(), "-"));- Long newNumber;- if (dryRun || retainIssueNumbers)- newNumber = oldNumber;- else- newNumber = OneDev.getInstance(IssueManager.class).getNextNumber(oneDevProject);- issue.setNumber(newNumber);- issueNumberMappings.put(oldNumber, newNumber);-- JsonNode fieldsNode = issueNode.get("fields");-- issue.setTitle(fieldsNode.get("summary").asText());-- String description = "";- JsonNode descriptionNode = fieldsNode.get("description");- if (descriptionNode != null && descriptionNode.hasNonNull("content")) {- description = getMarkdown(descriptionNode.get("content"), true);- if (StringUtils.isNotBlank("description"))- description = "#### Description\n\n" + description;- }-- JsonNode environmentNode = fieldsNode.get("environment");- if (environmentNode != null && environmentNode.hasNonNull("content")) {- String environment = getMarkdown(environmentNode.get("content"), true);- if (StringUtils.isNotBlank(environment)) {- if (StringUtils.isNotBlank(description))- description += "\n";- description += "#### Environment\n\n" + environment;- }- }-- if (StringUtils.isNotBlank(description))- issue.setDescription(description);-- String statusId = fieldsNode.get("status").get("id").asText();- String untranslatedStatusName = untranslatedStatusNames.get(statusId);- if (untranslatedStatusName != null) {- String statusName = statusMappings.get(untranslatedStatusName);- if (statusName != null) {- issue.setState(statusName);- } else {- unmappedIssueStatuses.add(untranslatedStatusName);- issue.setState(initialIssueState);- }- } else {- throw new ExplicitException("Can not find JIRA status of id: " + statusId);- }-- String typeId = fieldsNode.get("issuetype").get("id").asText();- String untranslatedTypeName = untranslatedTypeNames.get(typeId);- if (untranslatedTypeName != null) {- Pair<FieldSpec, String> mapped = typeMappings.get(untranslatedTypeName);- if (mapped != null) {- IssueField typeField = new IssueField();- typeField.setIssue(issue);- typeField.setName(mapped.getFirst().getName());- typeField.setType(InputSpec.ENUMERATION);- typeField.setValue(mapped.getSecond());- typeField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));- issue.getFields().add(typeField);- } else {- extraIssueInfo.put("Type", HtmlEscape.escapeHtml5(untranslatedTypeName));- unmappedIssueTypes.add(untranslatedTypeName);- }- } else {- String errorMessage = String.format("Can not find JIRA issue type by id (project: %s, issue type id: %s)",- jiraProject.get("name").asText(), typeId);- throw new ExplicitException(errorMessage);- }-- String priorityName = fieldsNode.get("priority").get("name").asText();- Pair<FieldSpec, String> mapped = priorityMappings.get(priorityName);- if (mapped != null) {- IssueField priorityField = new IssueField();- priorityField.setIssue(issue);- priorityField.setName(mapped.getFirst().getName());- priorityField.setType(InputSpec.ENUMERATION);- priorityField.setValue(mapped.getSecond());- priorityField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));- issue.getFields().add(priorityField);- } else {- extraIssueInfo.put("Priority", HtmlEscape.escapeHtml5(priorityName));- unmappedIssuePriorities.add(priorityName);- }-- List<String> components = new ArrayList<>();- for (JsonNode componentNode: fieldsNode.get("components"))- components.add(componentNode.get("name").asText());- if (!components.isEmpty())- extraIssueInfo.put("Components", joinAsMultilineHtml(components));-- List<String> affectedVersions = new ArrayList<>();- for (JsonNode versionNode: fieldsNode.get("versions"))- affectedVersions.add(versionNode.get("name").asText());- if (!affectedVersions.isEmpty())- extraIssueInfo.put("Affected Versions", joinAsMultilineHtml(affectedVersions));-- List<String> fixVersions = new ArrayList<>();- for (JsonNode versionNode: fieldsNode.get("fixVersions"))- fixVersions.add(versionNode.get("name").asText());- if (!fixVersions.isEmpty())- extraIssueInfo.put("Fix Versions", joinAsMultilineHtml(fixVersions));-- if (fieldsNode.hasNonNull("labels")) {- List<String> labels = new ArrayList<>();- for (JsonNode labelNode: fieldsNode.get("labels"))- labels.add(labelNode.asText());- if (!labels.isEmpty())- extraIssueInfo.put("Labels", joinAsMultilineHtml(labels));- }-- JsonNode authorNode = fieldsNode.get("reporter");- if (authorNode == null)- authorNode = fieldsNode.get("creator");- if (authorNode != null) {- User user = getUser(users, authorNode, logger);- if (user != null) {- issue.setSubmitter(user);- } else {- issue.setSubmitter(OneDev.getInstance(UserManager.class).getUnknown());- nonExistentLogins.add(authorNode.get("displayName").asText());- }- } else {- issue.setSubmitter(OneDev.getInstance(UserManager.class).getUnknown());- }-- if (fieldsNode.hasNonNull("assignee")) {- JsonNode assigneeNode = fieldsNode.get("assignee");- IssueField assigneeField = new IssueField();- assigneeField.setIssue(issue);- assigneeField.setName(importOption.getAssigneeIssueField());- assigneeField.setType(InputSpec.USER);-- User user = getUser(users, assigneeNode, logger);- if (user != null) {- assigneeField.setValue(user.getName());- issue.getFields().add(assigneeField);- } else {- nonExistentLogins.add(assigneeNode.get("displayName").asText());- }- }-- issue.setSubmitDate(ISODateTimeFormat.dateTime()- .parseDateTime(fieldsNode.get("created").asText())- .toDate());-- LastUpdate lastUpdate = new LastUpdate();- lastUpdate.setActivity("Opened");- lastUpdate.setDate(issue.getSubmitDate());- lastUpdate.setUser(issue.getSubmitter());- issue.setLastUpdate(lastUpdate);-- if (importOption.getDueDateIssueField() != null) {- String dueDate = fieldsNode.get("duedate").asText(null);- if (dueDate != null) {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(importOption.getDueDateIssueField());- issueField.setType(InputSpec.DATE);- issueField.setValue(dueDate);- issue.getFields().add(issueField);- }- }-- if (importOption.getTimeEstimateIssueField() != null) {- String timeEstimate = fieldsNode.get("timeestimate").asText(null);- if (timeEstimate != null) {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(importOption.getTimeEstimateIssueField());- issueField.setType(InputSpec.WORKING_PERIOD);- issueField.setValue(DateUtils.formatWorkingPeriod(Integer.valueOf(timeEstimate)/60));- issue.getFields().add(issueField);- }- }- if (importOption.getTimeSpentIssueField() != null) {- String timeSpent = fieldsNode.get("timespent").asText(null);- if (timeSpent != null) {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(importOption.getTimeSpentIssueField());- issueField.setType(InputSpec.WORKING_PERIOD);- issueField.setValue(DateUtils.formatWorkingPeriod(Integer.valueOf(timeSpent)/60));- issue.getFields().add(issueField);- }- }-- if (fieldsNode.hasNonNull("subtasks")) {- List<String> subtasks = new ArrayList<>();- for (JsonNode subtaskNode: fieldsNode.get("subtasks"))- subtasks.add(subtaskNode.get("key").asText());- if (!subtasks.isEmpty())- extraIssueInfo.put("Sub Tasks", StringUtils.join(subtasks, "<br>"));- }-- if (fieldsNode.hasNonNull("parent"))- extraIssueInfo.put("Parent Task", fieldsNode.get("parent").get("key").asText());-- if (fieldsNode.hasNonNull("issuelinks")) {- Map<String, List<String>> issueLinks = new LinkedHashMap<>();- for (JsonNode issueLinkNode: fieldsNode.get("issuelinks")) {- if (issueLinkNode.get("inwardIssue") != null) {- String type = issueLinkNode.get("type").get("inward").asText();- List<String> linkedIssues = issueLinks.get(type);- if (linkedIssues == null) {- linkedIssues = new ArrayList<>();- issueLinks.put(type, linkedIssues);- }- linkedIssues.add(issueLinkNode.get("inwardIssue").get("key").asText());- } else {- String type = issueLinkNode.get("type").get("outward").asText();- List<String> linkedIssues = issueLinks.get(type);- if (linkedIssues == null) {- linkedIssues = new ArrayList<>();- issueLinks.put(type, linkedIssues);- }- linkedIssues.add(issueLinkNode.get("outwardIssue").get("key").asText());- }- }- for (Map.Entry<String, List<String>> entry: issueLinks.entrySet())- extraIssueInfo.put(entry.getKey(), StringUtils.join(entry.getValue(), "<br>"));- }-- if (fieldsNode.hasNonNull("comment") && fieldsNode.get("comment").hasNonNull("comments")) {- for (JsonNode commentNode: fieldsNode.get("comment").get("comments")) {- String commentContent = getMarkdown(commentNode.get("body").get("content"), true);- if (StringUtils.isNotBlank(commentContent)) {- IssueComment comment = new IssueComment();- comment.setIssue(issue);- comment.setContent(commentContent);-- comment.setDate(ISODateTimeFormat.dateTime()- .parseDateTime(commentNode.get("created").asText())- .toDate());-- authorNode = commentNode.get("author");- if (authorNode != null) {- User user = getUser(users, authorNode, logger);- if (user != null) {- comment.setUser(user);- } else {- comment.setUser(OneDev.getInstance(UserManager.class).getUnknown());- nonExistentLogins.add(authorNode.get("displayName").asText());- }- } else {- comment.setUser(OneDev.getInstance(UserManager.class).getUnknown());- }-- issue.getComments().add(comment);- }- }- }-- issue.setCommentCount(issue.getComments().size());-- Set<String> fieldAndValues = new HashSet<>();- for (IssueField field: issue.getFields()) {- String fieldAndValue = field.getName() + "::" + field.getValue();- if (!fieldAndValues.add(fieldAndValue)) {- String errorMessage = String.format(- "Duplicate issue field mapping (issue: %s, field: %s)",- issueKey, fieldAndValue);- throw new ExplicitException(errorMessage);- }- }-- if (!extraIssueInfo.isEmpty()) {- StringBuilder builder = new StringBuilder("|");- for (String key: extraIssueInfo.keySet())- builder.append(key).append("|");- builder.append("\n|");- extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));- builder.append("\n|");- for (String value: extraIssueInfo.values())- builder.append(value).append("|");-- if (issue.getDescription() != null)- issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());- else- issue.setDescription(builder.toString());- }-- if (!dryRun) {- List<String> attachments = new ArrayList<>();-- if (fieldsNode.hasNonNull("attachment")) {- long maxUploadFileSize = OneDev.getInstance(SettingManager.class)- .getPerformanceSetting().getMaxUploadFileSize()*1L*1024*1024;- for (JsonNode attachmentNode: fieldsNode.get("attachment")) {- String attachmentName = attachmentNode.get("filename").asText();- int attachmentSize = attachmentNode.get("size").asInt();- if (attachmentSize > maxUploadFileSize) {- tooLargeAttachments.add(issueKey + ":" + attachmentName);- } else {- String endpoint = attachmentNode.get("content").asText();- WebTarget target = client.target(endpoint);- Invocation.Builder builder = target.request();- try (Response response = builder.get()) {- String errorMessage = JerseyUtils.checkStatus(endpoint, response);- if (errorMessage != null) {- throw new ExplicitException(String.format(- "Error downloading attachment (url: %s, error message: %s)",- endpoint, errorMessage));- }- try (InputStream is = response.readEntity(InputStream.class)) {- String oneDevAttachmentName = oneDevProject.saveAttachment(issue.getUUID(), attachmentName, is);- String oneDevAttachmentUrl = oneDevProject.getAttachmentUrlPath(issue.getUUID(), oneDevAttachmentName);- attachments.add("[" + oneDevAttachmentName + "](" + oneDevAttachmentUrl + ")");- } catch (IOException e) {- throw new RuntimeException(e);- }- }- }- }- }-- if (!attachments.isEmpty()) {- description = issue.getDescription();- if (StringUtils.isNotBlank(description))- description += "\n";- description += "#### Attachments\n" + Joiner.on(" • ").join(attachments);- issue.setDescription(description);- }- }-- issues.add(issue);- }- logger.log("Imported " + numOfImportedIssues.addAndGet(pageData.size()) + " issues");- }-- };-- String jiraProjectName = jiraProject.get("name").asText();- String jql = String.format("project = \"%s\" order by created DESC", jiraProjectName);- jql = URLEncoder.encode(jql, StandardCharsets.UTF_8.name());- apiEndpoint = server.getApiEndpoint("/search?jql=" + jql);- list(client, apiEndpoint, pageDataConsumer, logger);-- if (!dryRun) {- ReferenceMigrator migrator = new ReferenceMigrator(Issue.class, issueNumberMappings);- String jiraProjectKey = jiraProject.get("key").asText();- Dao dao = OneDev.getInstance(Dao.class);- for (Issue issue: issues) {- if (issue.getDescription() != null)- issue.setDescription(migrator.migratePrefixed(issue.getDescription(), jiraProjectKey + "-"));-- OneDev.getInstance(IssueManager.class).save(issue);- for (IssueSchedule schedule: issue.getSchedules())- dao.persist(schedule);- for (IssueField field: issue.getFields())- dao.persist(field);- for (IssueComment comment: issue.getComments()) {- comment.setContent(migrator.migratePrefixed(comment.getContent(), jiraProjectKey + "-"));- dao.persist(comment);- }- }- }-- ImportResult result = new ImportResult();- result.unmappedIssueStatuses.addAll(unmappedIssueStatuses);- result.unmappedIssueTypes.addAll(unmappedIssueTypes);- result.unmappedIssuePriorities.addAll(unmappedIssuePriorities);- result.nonExistentLogins.addAll(nonExistentLogins);- result.tooLargeAttachments.addAll(tooLargeAttachments);- result.errorAttachments.addAll(errorAttachments);-- return result;- } catch (UnsupportedEncodingException e) {- throw new RuntimeException(e);- } finally {- client.close();- }- }-- static ImportOption buildImportOption(ImportServer server, Collection<String> projectNames, TaskLogger logger) {- ImportOption importOption = new ImportOption();-- Map<String, JsonNode> projectNodes = getProjectNodes(server, logger);-- Client client = server.newClient();- try {- String apiEndpoint = server.getApiEndpoint("/status");- List<JsonNode> statusNodes = list(client, apiEndpoint, logger);-- Set<String> issueTypes = new LinkedHashSet<>();- Set<String> issueStatuses = new LinkedHashSet<>();- for (String projectName: projectNames) {- JsonNode projectNode = projectNodes.get(projectName);- if (projectNode == null)- throw new ExplicitException("Unable to find project: " + projectName);- apiEndpoint = server.getApiEndpoint("/issuetype/project?projectId=" + projectNode.get("id").asText());- for (JsonNode issueTypeNode: list(client, apiEndpoint, logger))- issueTypes.add(issueTypeNode.get("untranslatedName").asText());- if (projectNode.get("style").asText().equals("next-gen")) {- for (JsonNode statusNode: statusNodes) {- JsonNode scopeNode = statusNode.get("scope");- if (scopeNode != null && scopeNode.get("project").get("id").asText().equals(projectNode.get("id").asText()))- issueStatuses.add(statusNode.get("untranslatedName").asText());- }- } else {- for (JsonNode statusNode: statusNodes) {- if (statusNode.get("scope") == null)- issueStatuses.add(statusNode.get("untranslatedName").asText());- }- }- }- for (String issueType: issueTypes) {- IssueTypeMapping mapping = new IssueTypeMapping();- mapping.setJiraIssueType(issueType);- importOption.getIssueTypeMappings().add(mapping);- }- for (String issueStatus: issueStatuses) {- IssueStatusMapping mapping = new IssueStatusMapping();- mapping.setJiraIssueStatus(issueStatus);- importOption.getIssueStatusMappings().add(mapping);- }- apiEndpoint = server.getApiEndpoint("/priority");- for (JsonNode priorityNode: list(client, apiEndpoint, logger)) {- IssuePriorityMapping mapping = new IssuePriorityMapping();- mapping.setJiraIssuePriority(priorityNode.get("name").asText());- importOption.getIssuePriorityMappings().add(mapping);- }- } finally {- client.close();- }- return importOption;- }-- static Map<String, JsonNode> getProjectNodes(ImportServer where, TaskLogger logger) {- Map<String, JsonNode> projectNodes = new LinkedHashMap<>();-- Client client = where.newClient();- try {- for (JsonNode projectNode: list(client, where.getApiEndpoint("/project/search"), logger))- projectNodes.put(projectNode.get("name").asText(), projectNode);- } finally {- client.close();- }- return projectNodes;- }-- static List<JsonNode> list(Client client, String apiEndpoint, TaskLogger logger) {- List<JsonNode> result = new ArrayList<>();- list(client, apiEndpoint, new PageDataConsumer() {-- @Override- public void consume(List<JsonNode> pageData) {- result.addAll(pageData);- }-- }, logger);- return result;- }-- static void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer,- TaskLogger logger) {- URI uri;- try {- uri = new URIBuilder(apiEndpoint)- .addParameter("maxResults", String.valueOf(PER_PAGE)).build();- } catch (URISyntaxException e) {- throw new RuntimeException(e);- }-- int page = 0;- while (true) {- try {- URIBuilder builder = new URIBuilder(uri);- builder.addParameter("startAt", String.valueOf(page*PER_PAGE));- List<JsonNode> pageData = new ArrayList<>();-- JsonNode resultNode = JerseyUtils.get(client, builder.build().toString(), logger);- if (resultNode.isArray()) {- for (JsonNode each: resultNode)- pageData.add(each);- } else {- if (resultNode.hasNonNull("values")) {- for (JsonNode each: resultNode.get("values"))- pageData.add(each);- } else if (resultNode.hasNonNull("issues")) {- for (JsonNode each: resultNode.get("issues"))- pageData.add(each);- } else if (resultNode.hasNonNull("comments")) {- for (JsonNode each: resultNode.get("comments"))- pageData.add(each);- }- }- pageDataConsumer.consume(pageData);- if (pageData.size() < PER_PAGE)- break;- page++;- } catch (URISyntaxException | InterruptedException e) {- throw new RuntimeException(e);- }- }- }--}diff --git a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/IssueImportSource.java b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/IssueImportSource.javadeleted file mode 100644index f1cff0927d..0000000000--- a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/IssueImportSource.java+++ /dev/null@@ -1,68 +0,0 @@-package io.onedev.server.plugin.imports.jiracloud;--import java.io.Serializable;-import java.util.ArrayList;-import java.util.Collections;-import java.util.List;--import javax.ws.rs.client.Client;--import org.hibernate.validator.constraints.NotEmpty;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--import com.fasterxml.jackson.databind.JsonNode;--import io.onedev.commons.utils.TaskLogger;-import io.onedev.server.web.editable.annotation.ChoiceProvider;-import io.onedev.server.web.editable.annotation.Editable;-import io.onedev.server.web.util.WicketUtils;--@Editable-public class IssueImportSource implements Serializable {-- private static final long serialVersionUID = 1L;-- private static final Logger logger = LoggerFactory.getLogger(IssueImportSource.class);-- private String project;-- @Editable(order=400, name="JIRA Project", description="Choose JIRA project to import issues from")- @ChoiceProvider("getProjectChoices")- @NotEmpty- public String getProject() {- return project;- }-- public void setProject(String project) {- this.project = project;- }-- @SuppressWarnings("unused")- private static List<String> getProjectChoices() {- List<String> choices = new ArrayList<>();-- ImportServer server = WicketUtils.getPage().getMetaData(ImportServer.META_DATA_KEY);-- Client client = server.newClient();- try {- String apiEndpoint = server.getApiEndpoint("/project/search");- for (JsonNode projectNode: ImportUtils.list(client, apiEndpoint, new TaskLogger() {-- @Override- public void log(String message, String sessionId) {- logger.info(message);- }-- })) {- choices.add(projectNode.get("name").asText());- }- } finally {- client.close();- }-- Collections.sort(choices);- return choices;- }--}diff --git a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/JiraIssueImporter.java b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/JiraIssueImporter.javaindex b0855cd526..164f8c03f5 100644--- a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/JiraIssueImporter.java+++ b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/JiraIssueImporter.java@@ -1,58 +1,86 @@package io.onedev.server.plugin.imports.jiracloud;-import java.util.HashMap;-import java.util.Map;-import java.util.Optional;+import java.io.Serializable;+import java.util.List;-import javax.ws.rs.client.Client;--import com.fasterxml.jackson.databind.JsonNode;import com.google.common.collect.Lists;-import io.onedev.commons.utils.ExplicitException;import io.onedev.commons.utils.TaskLogger;import io.onedev.server.imports.IssueImporter;import io.onedev.server.model.Project;-import io.onedev.server.model.User;-import io.onedev.server.web.util.WicketUtils;+import io.onedev.server.web.util.ImportStep;-public class JiraIssueImporter extends IssueImporter<ImportServer, IssueImportSource, ImportOption> {+public class JiraIssueImporter implements IssueImporter {private static final long serialVersionUID = 1L;- @Override- public String getName() {- return ImportUtils.NAME;- }+ private final ImportStep<ImportServer> serverStep = new ImportStep<ImportServer>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Authenticate to JIRA cloud";+ }++ @Override+ protected ImportServer newSetting() {+ return new ImportServer();+ }++ };++ private final ImportStep<ImportProject> projectStep = new ImportStep<ImportProject>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Choose project";+ }++ @Override+ protected ImportProject newSetting() {+ ImportProject project = new ImportProject();+ project.server = serverStep.getSetting();+ return project;+ }++ };++ private final ImportStep<ImportOption> optionStep = new ImportStep<ImportOption>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Specify import option";+ }+ @Override+ protected ImportOption newSetting() {+ return serverStep.getSetting().buildImportOption(+ Lists.newArrayList(projectStep.getSetting().getProject()));+ }++ };+@Override- public IssueImportSource getWhat(ImportServer where, TaskLogger logger) {- WicketUtils.getPage().setMetaData(ImportServer.META_DATA_KEY, where);- return new IssueImportSource();+ public String getName() {+ return JiraPluginModule.NAME;}@Override- public ImportOption getHow(ImportServer where, IssueImportSource what, TaskLogger logger) {- return ImportUtils.buildImportOption(where, Lists.newArrayList(what.getProject()), logger);+ public String doImport(Project project, boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {+ ImportServer server = serverStep.getSetting();+ String jiraProject = projectStep.getSetting().getProject();+ ImportOption option = optionStep.getSetting();+ return server.importIssues(project, jiraProject, option, retainIssueNumbers, dryRun, logger);}@Override- public String doImport(ImportServer where, IssueImportSource what, ImportOption how,- Project project, boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {- Map<String, JsonNode> projectNodes = ImportUtils.getProjectNodes(where, logger);-- Client client = where.newClient();- try {- Map<String, Optional<User>> users = new HashMap<>();- JsonNode projectNode = projectNodes.get(what.getProject());- if (projectNode == null)- throw new ExplicitException("Unable to find project: " + what.getProject());- ImportResult result = ImportUtils.importIssues(where, projectNode, project, retainIssueNumbers,- how, users, dryRun, logger);- return result.toHtml("Issues imported successfully");- } finally {- client.close();- }+ public List<ImportStep<? extends Serializable>> getSteps() {+ return Lists.newArrayList(serverStep, projectStep, optionStep);}}diff --git a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/JiraPluginModule.java b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/JiraPluginModule.javaindex 386e3c8540..d2dbe40475 100644--- a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/JiraPluginModule.java+++ b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/JiraPluginModule.java@@ -1,6 +1,5 @@package io.onedev.server.plugin.imports.jiracloud;-import java.io.Serializable;import java.util.Collection;import com.google.common.collect.Lists;@@ -17,6 +16,8 @@ import io.onedev.server.imports.ProjectImporterContribution;*/public class JiraPluginModule extends AbstractPluginModule {+ static final String NAME = "JIRA Cloud";+@Overrideprotected void configure() {super.configure();@@ -25,7 +26,7 @@ public class JiraPluginModule extends AbstractPluginModule {contribute(ProjectImporterContribution.class, new ProjectImporterContribution() {@Override- public Collection<ProjectImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> getImporters() {+ public Collection<ProjectImporter> getImporters() {return Lists.newArrayList(new JiraProjectImporter());}@@ -39,7 +40,7 @@ public class JiraPluginModule extends AbstractPluginModule {contribute(IssueImporterContribution.class, new IssueImporterContribution() {@Override- public Collection<IssueImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> getImporters() {+ public Collection<IssueImporter> getImporters() {return Lists.newArrayList(new JiraIssueImporter());}diff --git a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/JiraProjectImporter.java b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/JiraProjectImporter.javaindex a79e5532fc..baf4c3e42e 100644--- a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/JiraProjectImporter.java+++ b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/JiraProjectImporter.java@@ -1,109 +1,90 @@package io.onedev.server.plugin.imports.jiracloud;-import java.util.ArrayList;-import java.util.Collection;-import java.util.HashMap;+import java.io.Serializable;import java.util.List;-import java.util.Map;-import java.util.Optional;import java.util.stream.Collectors;-import javax.ws.rs.client.Client;+import com.google.common.collect.Lists;-import com.fasterxml.jackson.databind.JsonNode;-import com.google.common.base.Preconditions;--import io.onedev.commons.utils.ExplicitException;import io.onedev.commons.utils.TaskLogger;-import io.onedev.server.OneDev;-import io.onedev.server.entitymanager.ProjectManager;import io.onedev.server.imports.ProjectImporter;-import io.onedev.server.model.Project;-import io.onedev.server.model.User;-import io.onedev.server.storage.StorageManager;-import io.onedev.server.util.JerseyUtils;+import io.onedev.server.web.util.ImportStep;-public class JiraProjectImporter extends ProjectImporter<ImportServer, ProjectImportSource, ImportOption> {+public class JiraProjectImporter implements ProjectImporter {private static final long serialVersionUID = 1L;++ private final ImportStep<ImportServer> serverStep = new ImportStep<ImportServer>() {- @Override- public String getName() {- return ImportUtils.NAME;- }+ private static final long serialVersionUID = 1L;- @Override- public ProjectImportSource getWhat(ImportServer where, TaskLogger logger) {- ProjectImportSource importSource = new ProjectImportSource();- for (String projectName: ImportUtils.getProjectNodes(where, logger).keySet()) {- ProjectMapping mapping = new ProjectMapping();- mapping.setJiraProject(projectName);- mapping.setOneDevProject(projectName.replace(' ', '-'));- importSource.getProjectMappings().add(mapping);+ @Override+ public String getTitle() {+ return "Authenticate to JIRA cloud";+ }++ @Override+ protected ImportServer newSetting() {+ return new ImportServer();+ }++ };++ private final ImportStep<ImportProjects> projectsStep = new ImportStep<ImportProjects>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Specify projects";+ }++ @Override+ protected ImportProjects newSetting() {+ ImportProjects projects = new ImportProjects();+ for (String project: serverStep.getSetting().listProjects()) {+ ProjectMapping mapping = new ProjectMapping();+ mapping.setJiraProject(project);+ mapping.setOneDevProject(project.replace(' ', '-'));+ projects.getProjectMappings().add(mapping);+ }+ return projects;+ }++ };++ private final ImportStep<ImportOption> optionStep = new ImportStep<ImportOption>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Specify import option";+ }++ @Override+ protected ImportOption newSetting() {+ List<String> jiraProjects = projectsStep.getSetting().getProjectMappings().stream()+ .map(it->it.getJiraProject()).collect(Collectors.toList());+ return serverStep.getSetting().buildImportOption(jiraProjects);}- return importSource;++ };++ @Override+ public String getName() {+ return JiraPluginModule.NAME;}@Override- public ImportOption getHow(ImportServer where, ProjectImportSource what, TaskLogger logger) {- List<String> jiraProjects = what.getProjectMappings().stream()- .map(it->it.getJiraProject()).collect(Collectors.toList());- return ImportUtils.buildImportOption(where, jiraProjects, logger);+ public String doImport(boolean dryRun, TaskLogger logger) {+ return serverStep.getSetting().importProjects(+ projectsStep.getSetting(), optionStep.getSetting(), dryRun, logger);}@Override- public String doImport(ImportServer where, ProjectImportSource what, ImportOption how,- boolean dryRun, TaskLogger logger) {- Collection<Long> projectIds = new ArrayList<>();-- Map<String, JsonNode> projectNodes = ImportUtils.getProjectNodes(where, logger);-- Client client = where.newClient();- try {- Map<String, Optional<User>> users = new HashMap<>();- ImportResult result = new ImportResult();-- for (ProjectMapping projectMapping: what.getProjectMappings()) {- String jiraProject = projectMapping.getJiraProject();- JsonNode projectNode = projectNodes.get(jiraProject);- if (projectNode == null)- throw new ExplicitException("Unable to find project: " + jiraProject);-- String apiEndpoint = where.getApiEndpoint("/project/" + projectNode.get("id").asText());-- // Get more detail project information- projectNode = JerseyUtils.get(client, apiEndpoint, logger);-- ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);- Project project = projectManager.initialize(projectMapping.getOneDevProject());- Preconditions.checkState(project.isNew());-- project.setDescription(projectNode.get("description").asText(null));-- if (!dryRun) {- OneDev.getInstance(ProjectManager.class).create(project);- projectIds.add(project.getId());- }-- logger.log("Importing issues from project " + jiraProject + "...");- ImportResult currentResult = ImportUtils.importIssues(where, projectNode,- project, true, how, users, dryRun, logger);- result.nonExistentLogins.addAll(currentResult.nonExistentLogins);- result.errorAttachments.addAll(currentResult.errorAttachments);- result.tooLargeAttachments.addAll(currentResult.tooLargeAttachments);- result.unmappedIssuePriorities.addAll(currentResult.unmappedIssuePriorities);- result.unmappedIssueStatuses.addAll(currentResult.unmappedIssueStatuses);- result.unmappedIssueTypes.addAll(currentResult.unmappedIssueTypes);- }-- return result.toHtml("Projects imported successfully");- } catch (Exception e) {- for (Long projectId: projectIds)- OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);- throw new RuntimeException(e);- } finally {- client.close();- }+ public List<ImportStep<? extends Serializable>> getSteps() {+ return Lists.newArrayList(serverStep, projectsStep, optionStep);}}diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/IssueImportSource.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportProject.javasimilarity index 56%rename from server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/IssueImportSource.javarename to server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportProject.javaindex bff3d7a5cb..f4cf37f4b7 100644--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/IssueImportSource.java+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportProject.java@@ -5,20 +5,21 @@ import java.util.List;import org.hibernate.validator.constraints.NotEmpty;+import io.onedev.server.util.ComponentContext;+import io.onedev.server.web.editable.BeanEditor;import io.onedev.server.web.editable.annotation.ChoiceProvider;import io.onedev.server.web.editable.annotation.Editable;-import io.onedev.server.web.util.WicketUtils;@Editable-public class IssueImportSource implements Serializable {+public class ImportProject implements Serializable {private static final long serialVersionUID = 1L;+ ImportServer server;+private String project;- @Editable(order=100, name="YouTrack Project", description="Choose YouTrack project to "- + "import issues from. Available projects will be populated when above options "- + "are specified")+ @Editable(order=400, name="YouTrack Project", description="Choose YouTrack project to import issues from")@ChoiceProvider("getProjectChoices")@NotEmptypublic String getProject() {@@ -31,7 +32,9 @@ public class IssueImportSource implements Serializable {@SuppressWarnings("unused")private static List<String> getProjectChoices() {- return WicketUtils.getPage().getMetaData(ImportServer.META_DATA_KEY).getProjectChoices();+ BeanEditor editor = ComponentContext.get().getComponent().findParent(BeanEditor.class);+ ImportProject setting = (ImportProject) editor.getModelObject();+ return setting.server.listProjects();}}diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ProjectImportSource.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportProjects.javasimilarity index 93%rename from server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ProjectImportSource.javarename to server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportProjects.javaindex c399f9d421..8a4635e834 100644--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ProjectImportSource.java+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportProjects.java@@ -15,16 +15,20 @@ import io.onedev.server.model.Project;import io.onedev.server.util.validation.Validatable;import io.onedev.server.util.validation.annotation.ClassValidating;import io.onedev.server.web.editable.annotation.Editable;+import io.onedev.server.web.editable.annotation.OmitName;@Editable@ClassValidating-public class ProjectImportSource implements Serializable, Validatable {+public class ImportProjects implements Serializable, Validatable {private static final long serialVersionUID = 1L;+ ImportServer server;+private List<ProjectMapping> projectMappings = new ArrayList<>();@Editable(order=100, name="Projects to Import")+ @OmitName@Size(min=1, max=10000, message="No projects to import")public List<ProjectMapping> getProjectMappings() {return projectMappings;diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportServer.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportServer.javaindex 04bc86fa68..1fe86b0701 100644--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportServer.java+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportServer.java@@ -1,10 +1,25 @@package io.onedev.server.plugin.imports.youtrack;+import java.io.IOException;+import java.io.InputStream;import java.io.Serializable;+import java.net.URI;+import java.net.URISyntaxException;import java.util.ArrayList;+import java.util.Collection;import java.util.Collections;+import java.util.Date;+import java.util.HashMap;+import java.util.HashSet;+import java.util.LinkedHashMap;+import java.util.LinkedHashSet;import java.util.List;+import java.util.Map;+import java.util.Set;+import java.util.concurrent.atomic.AtomicInteger;+import java.util.stream.Collectors;+import javax.annotation.Nullable;import javax.validation.ConstraintValidatorContext;import javax.ws.rs.client.Client;import javax.ws.rs.client.ClientBuilder;@@ -12,17 +27,58 @@ import javax.ws.rs.client.Invocation;import javax.ws.rs.client.WebTarget;import javax.ws.rs.core.Response;+import org.apache.http.client.utils.URIBuilder;import org.apache.wicket.MetaDataKey;import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;import org.hibernate.validator.constraints.NotEmpty;import org.slf4j.Logger;import org.slf4j.LoggerFactory;+import org.unbescape.html.HtmlEscape;import com.fasterxml.jackson.databind.JsonNode;+import com.google.common.base.Preconditions;+import com.google.common.collect.Sets;+import io.onedev.commons.utils.ExceptionUtils;+import io.onedev.commons.utils.ExplicitException;import io.onedev.commons.utils.StringUtils;import io.onedev.commons.utils.TaskLogger;+import io.onedev.server.OneDev;+import io.onedev.server.entitymanager.IssueLinkManager;+import io.onedev.server.entitymanager.IssueManager;+import io.onedev.server.entitymanager.LinkSpecManager;+import io.onedev.server.entitymanager.ProjectManager;+import io.onedev.server.entitymanager.SettingManager;+import io.onedev.server.entitymanager.UserManager;+import io.onedev.server.entityreference.ReferenceMigrator;+import io.onedev.server.model.Issue;+import io.onedev.server.model.IssueComment;+import io.onedev.server.model.IssueField;+import io.onedev.server.model.IssueLink;+import io.onedev.server.model.IssueSchedule;+import io.onedev.server.model.LinkSpec;+import io.onedev.server.model.Project;+import io.onedev.server.model.User;+import io.onedev.server.model.support.LastUpdate;+import io.onedev.server.model.support.administration.GlobalIssueSetting;+import io.onedev.server.model.support.inputspec.InputSpec;+import io.onedev.server.model.support.issue.StateSpec;+import io.onedev.server.model.support.issue.field.spec.ChoiceField;+import io.onedev.server.model.support.issue.field.spec.DateField;+import io.onedev.server.model.support.issue.field.spec.DateTimeField;+import io.onedev.server.model.support.issue.field.spec.FieldSpec;+import io.onedev.server.model.support.issue.field.spec.FloatField;+import io.onedev.server.model.support.issue.field.spec.IntegerField;+import io.onedev.server.model.support.issue.field.spec.TextField;+import io.onedev.server.model.support.issue.field.spec.UserChoiceField;+import io.onedev.server.model.support.issue.field.spec.WorkingPeriodField;+import io.onedev.server.persistence.dao.Dao;+import io.onedev.server.security.SecurityUtils;+import io.onedev.server.storage.StorageManager;+import io.onedev.server.util.DateUtils;import io.onedev.server.util.JerseyUtils;+import io.onedev.server.util.Pair;+import io.onedev.server.util.JerseyUtils.PageDataConsumer;import io.onedev.server.util.validation.Validatable;import io.onedev.server.util.validation.annotation.ClassValidating;import io.onedev.server.web.editable.annotation.Editable;@@ -36,6 +92,8 @@ public class ImportServer implements Serializable, Validatable {private static final Logger logger = LoggerFactory.getLogger(ImportServer.class);+ private static final int PER_PAGE = 50;+static final MetaDataKey<ImportServer> META_DATA_KEY = new MetaDataKey<ImportServer>() {private static final long serialVersionUID = 1L;@@ -90,6 +148,1027 @@ public class ImportServer implements Serializable, Validatable {return StringUtils.stripEnd(apiUrl, "/") + "/" + StringUtils.stripStart(apiPath, "/");}+ List<String> listProjects() {+ List<String> projects = new ArrayList<>();++ Client client = newClient();+ try {+ TaskLogger logger = new TaskLogger() {++ @Override+ public void log(String message, String sessionId) {+ ImportServer.logger.info(message);+ }++ };+ String apiEndpoint = getApiEndpoint("/admin/projects?fields=name");+ for (JsonNode projectNode: list(client, apiEndpoint, logger))+ projects.add(projectNode.get("name").asText());+ } finally {+ client.close();+ }++ Collections.sort(projects);+ return projects;+ }++ private Client newClient() {+ Client client = ClientBuilder.newClient();+ client.register(HttpAuthenticationFeature.basic(getUserName(), getPassword()));+ return client;+ }++ ImportOption buildImportOption(ImportProjects projects) {+ Client client = newClient();+ try {+ List<JsonNode> projectNodes = new ArrayList<>();+ String apiEndpoint = getApiEndpoint("/admin/projects?fields=id,name,customFields(field(name),bundle(values(name)))");+ Set<String> projectNames = projects.getProjectMappings().stream()+ .map(it->it.getYouTrackProject()).collect(Collectors.toSet());+ for (JsonNode projectNode: list(client, apiEndpoint, new TaskLogger() {++ @Override+ public void log(String message, String sessionId) {+ logger.info(message);+ }++ })) {+ if (projectNames.contains(projectNode.get("name").asText()))+ projectNodes.add(projectNode);+ }++ return buildImportOption(projectNodes);+ } finally {+ client.close();+ }+ }++ ImportOption buildImportOption(String project) {+ Client client = newClient();+ try {+ String apiEndpoint = getApiEndpoint("/admin/projects?fields=id,name,customFields(field(name),bundle(values(name)))");+ for (JsonNode projectNode: list(client, apiEndpoint, new TaskLogger() {++ @Override+ public void log(String message, String sessionId) {+ logger.info(message);+ }++ })) {+ if (project.equals(projectNode.get("name").asText())) {+ List<JsonNode> projectNodes = new ArrayList<>();+ projectNodes.add(projectNode);+ return buildImportOption(projectNodes);+ }+ }+ throw new ExplicitException("Unable to find YouTrack project: " + project);+ } finally {+ client.close();+ }+ }++ ImportOption buildImportOption(Collection<JsonNode> projectNodes) {+ ImportOption option = new ImportOption();+ Client client = newClient();+ try {+ Set<String> youTrackIssueStates = new LinkedHashSet<>();+ Set<String> youTrackIssueFields = new LinkedHashSet<>();++ for (JsonNode projectNode: projectNodes) {+ for (JsonNode customFieldNode: projectNode.get("customFields")) {+ String fieldName = customFieldNode.get("field").get("name").asText();+ JsonNode bundleNode = customFieldNode.get("bundle");+ if (bundleNode != null) {+ String bundleType = bundleNode.get("$type").asText();+ if (bundleType.equals("StateBundle")) {+ for (JsonNode valueNode: bundleNode.get("values"))+ youTrackIssueStates.add(valueNode.get("name").asText());+ } else if (bundleType.equals("EnumBundle") || bundleType.equals("OwnedBundle")) {+ for (JsonNode valueNode: bundleNode.get("values"))+ youTrackIssueFields.add(fieldName + "::" + valueNode.get("name").asText());+ } else {+ youTrackIssueFields.add(fieldName);+ }+ } else {+ youTrackIssueFields.add(fieldName);+ }+ }+ }++ for (String youTrackIssueState: youTrackIssueStates) {+ IssueStateMapping mapping = new IssueStateMapping();+ mapping.setYouTrackIssueState(youTrackIssueState);+ option.getIssueStateMappings().add(mapping);+ }++ for (String youTrackIssueField: youTrackIssueFields) {+ IssueFieldMapping mapping = new IssueFieldMapping();+ mapping.setYouTrackIssueField(youTrackIssueField);+ option.getIssueFieldMappings().add(mapping);+ }++ TaskLogger taskLogger = new TaskLogger() {++ @Override+ public void log(String message, String sessionId) {+ logger.info(message);+ }++ };+ String apiEndpoint = getApiEndpoint("/issueTags?fields=name");+ for (JsonNode tagNode: list(client, apiEndpoint, taskLogger)) {+ IssueTagMapping mapping = new IssueTagMapping();+ mapping.setYouTrackIssueTag(tagNode.get("name").asText());+ option.getIssueTagMappings().add(mapping);+ }++ apiEndpoint = getApiEndpoint("/issueLinkTypes?fields=name");+ for (JsonNode tagNode: list(client, apiEndpoint, taskLogger)) {+ IssueLinkMapping mapping = new IssueLinkMapping();+ mapping.setYouTrackIssueLink(tagNode.get("name").asText());+ option.getIssueLinkMappings().add(mapping);+ }++ } finally {+ client.close();+ }+ return option;+ }++ private ImportResult importIssues(String youTrackProjectId, Project oneDevProject, boolean retainIssueNumbers,+ ImportOption option, boolean dryRun, TaskLogger logger) {+ Client client = newClient();+ try {+ String apiEndpoint = getApiEndpoint("/admin/projects/" + youTrackProjectId + "?fields=shortName");+ String youTrackProjectShortName = JerseyUtils.get(client, apiEndpoint, logger).get("shortName").asText();++ Map<String, String> stateMappings = new HashMap<>();+ Map<String, Pair<FieldSpec, String>> fieldMappings = new HashMap<>();+ Map<String, Pair<FieldSpec, String>> tagMappings = new HashMap<>();+ Map<String, LinkSpec> linkMappings = new HashMap<>();++ Map<Long, Long> issueNumberMappings = new HashMap<>();+ Map<Long, Issue> issueMappings = new HashMap<>();++ for (IssueStateMapping mapping: option.getIssueStateMappings())+ stateMappings.put(mapping.getYouTrackIssueState(), mapping.getOneDevIssueState());++ for (IssueFieldMapping mapping: option.getIssueFieldMappings()) {+ String oneDevFieldName;+ String oneDevFieldValue;+ if (mapping.getOneDevIssueField().contains("::")) {+ oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");+ oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");+ } else {+ oneDevFieldName = mapping.getOneDevIssueField();+ oneDevFieldValue = null;+ }+ FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);+ if (fieldSpec == null)+ throw new ExplicitException("No field spec found: " + oneDevFieldName);+ fieldMappings.put(mapping.getYouTrackIssueField(), new Pair<>(fieldSpec, oneDevFieldValue));+ }+ for (IssueTagMapping mapping: option.getIssueTagMappings()) {+ String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");+ String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");+ FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);+ if (fieldSpec == null)+ throw new ExplicitException("No field spec found: " + oneDevFieldName);+ tagMappings.put(mapping.getYouTrackIssueTag(), new Pair<>(fieldSpec, oneDevFieldValue));+ }+ for (IssueLinkMapping mapping: option.getIssueLinkMappings()) {+ LinkSpec linkSpec = OneDev.getInstance(LinkSpecManager.class).find(mapping.getOneDevIssueLink());+ if (linkSpec == null)+ throw new ExplicitException("No link spec found: " + mapping.getOneDevIssueLink());+ linkMappings.put(mapping.getYouTrackIssueLink(), linkSpec);+ }++ Set<String> nonExistentLogins = new LinkedHashSet<>();+ Set<String> unmappedIssueTags = new LinkedHashSet<>();+ Set<String> unmappedIssueFields = new LinkedHashSet<>();+ Set<String> unmappedIssueLinks = new LinkedHashSet<>();+ Set<String> unmappedIssueStates = new LinkedHashSet<>();+ Map<String, String> mismatchedIssueFields = new LinkedHashMap<>();+ Set<String> tooLargeAttachments = new LinkedHashSet<>();++ AtomicInteger numOfImportedIssues = new AtomicInteger(0);++ List<Issue> issues = new ArrayList<>();+ Map<Long, Pair<LinkSpec, List<Long>>> issueLinkInfo = new HashMap<>();+ Map<Long, Set<Set<Long>>> processedSymmetricLinks = new HashMap<>();++ String fields = ""+ + "idReadable,"+ + "numberInProject,"+ + "summary,"+ + "description,"+ + "created,"+ + "comments(created,text,author(login,name,email),attachments(name,url,size)),"+ + "attachments(name,url,comment,size),"+ + "reporter(login,name,email),"+ + "tags(name),"+ + "customFields(name,value(name,login,email,presentation,text),projectCustomField(field(fieldType(id)))),"+ + "links(direction,linkType(name,sourceToTarget,targetToSource),issues(numberInProject))";+ PageDataConsumer pageDataConsumer = new PageDataConsumer() {++ @Nullable+ private String processAttachments(String issueUUID, String readableIssueId, @Nullable String markdown,+ List<JsonNode> attachmentNodes, Set<String> tooLargeAttachments) {+ if (markdown == null)+ markdown = "";++ Map<String, String> unreferencedAttachments = new LinkedHashMap<>();++ long maxUploadFileSize = OneDev.getInstance(SettingManager.class)+ .getPerformanceSetting().getMaxUploadFileSize()*1L*1024*1024;+ for (JsonNode attachmentNode: attachmentNodes) {+ String attachmentName = attachmentNode.get("name").asText(null);+ String attachmentUrl = attachmentNode.get("url").asText(null);+ long attachmentSize = attachmentNode.get("size").asLong(0);+ if (attachmentSize != 0 && attachmentName != null && attachmentUrl != null) {+ if (attachmentSize > maxUploadFileSize) {+ tooLargeAttachments.add(readableIssueId + ":" + attachmentName);+ } else {+ if (!attachmentUrl.startsWith("/api"))+ throw new ExplicitException("Unexpected attachment url: " + attachmentUrl);+ attachmentUrl = attachmentUrl.substring("/api".length());++ String endpoint = getApiEndpoint(attachmentUrl);+ WebTarget target = client.target(endpoint);+ Invocation.Builder builder = target.request();+ try (Response response = builder.get()) {+ String errorMessage = JerseyUtils.checkStatus(endpoint, response);+ if (errorMessage != null) {+ throw new ExplicitException(String.format(+ "Error downloading attachment (url: %s, error message: %s)",+ endpoint, errorMessage));+ }+ try (InputStream is = response.readEntity(InputStream.class)) {+ String oneDevAttachmentName = oneDevProject.saveAttachment(issueUUID, attachmentName, is);+ String oneDevAttachmentUrl = oneDevProject.getAttachmentUrlPath(issueUUID, oneDevAttachmentName);+ if (markdown.contains("(" + attachmentName + ")")) {+ markdown = markdown.replace("(" + attachmentName + ")", "(" + oneDevAttachmentUrl + ")");+ } else {+ unreferencedAttachments.put(attachmentName, oneDevAttachmentUrl);+ }+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ }+ }+ }+ }++ if (!unreferencedAttachments.isEmpty()) {+ markdown += "\n\n";+ for (Map.Entry<String, String> entry: unreferencedAttachments.entrySet())+ markdown += "[" + entry.getKey() + "](" + entry.getValue() + ") ";+ }++ if (markdown.length() == 0)+ markdown = null;++ return markdown;+ }++ private String joinAsMultilineHtml(List<String> values) {+ List<String> escapedValues = new ArrayList<>();+ for (String value: values)+ escapedValues.add(HtmlEscape.escapeHtml5(value));+ return StringUtils.join(escapedValues, "<br>");+ }++ @Nullable+ private String getEmail(JsonNode userNode) {+ JsonNode emailNode = userNode.get("email");+ if (emailNode != null)+ return emailNode.asText(null);+ else+ return null;+ }++ @Override+ public void consume(List<JsonNode> pageData) throws InterruptedException {+ for (JsonNode issueNode: pageData) {+ if (Thread.interrupted())+ throw new InterruptedException();++ Issue issue = new Issue();+ String readableId = issueNode.get("idReadable").asText();+ Long oldNumber = issueNode.get("numberInProject").asLong();+ Long newNumber;+ if (dryRun || retainIssueNumbers)+ newNumber = oldNumber;+ else+ newNumber = getIssueManager().getNextNumber(oneDevProject);+ issue.setNumber(newNumber);+ issueNumberMappings.put(oldNumber, newNumber);+ issueMappings.put(oldNumber, issue);+ issue.setTitle(issueNode.get("summary").asText());+ issue.setDescription(issueNode.get("description").asText(null));+ issue.setSubmitDate(new Date(issueNode.get("created").asLong(System.currentTimeMillis())));+ issue.setProject(oneDevProject);+ issue.setNumberScope(oneDevProject.getForkRoot());++ if (!dryRun) {+ List<JsonNode> attachmentNodes = new ArrayList<>();+ for (JsonNode attachmentNode: issueNode.get("attachments")) {+ if (!attachmentNode.hasNonNull("comment"))+ attachmentNodes.add(attachmentNode);+ }+ issue.setDescription(processAttachments(issue.getUUID(), readableId,+ issue.getDescription(), attachmentNodes, tooLargeAttachments));+ }++ Map<String, String> extraIssueInfo = new LinkedHashMap<>();++ if (issueNode.hasNonNull("reporter")) {+ JsonNode reporterNode = issueNode.get("reporter");+ String email = getEmail(reporterNode);+ String login = reporterNode.get("login").asText();+ if (email != null) {+ User user = OneDev.getInstance(UserManager.class).findByVerifiedEmailAddress(email);+ if (user != null) {+ issue.setSubmitter(user);+ } else {+ issue.setSubmitter(OneDev.getInstance(UserManager.class).getUnknown());+ nonExistentLogins.add(login);+ }+ } else {+ issue.setSubmitter(OneDev.getInstance(UserManager.class).getUnknown());+ nonExistentLogins.add(login);+ }+ } else {+ issue.setSubmitter(SecurityUtils.getUser());+ }++ LastUpdate lastUpdate = new LastUpdate();+ lastUpdate.setActivity("Opened");+ lastUpdate.setDate(issue.getSubmitDate());+ lastUpdate.setUser(issue.getSubmitter());+ issue.setLastUpdate(lastUpdate);++ StateSpec initialState = getIssueSetting().getInitialStateSpec();+ for (JsonNode customFieldNode: issueNode.get("customFields")) {+ String fieldName = customFieldNode.get("name").asText();+ String fieldType = customFieldNode.get("$type").asText();+ switch (fieldType) {+ case "StateIssueCustomField":+ if (customFieldNode.hasNonNull("value")) {+ String state = customFieldNode.get("value").get("name").asText();+ String mappedState = stateMappings.get(state);+ if (mappedState != null)+ issue.setState(mappedState);+ else+ unmappedIssueStates.add(state);+ }+ break;+ case "SingleBuildIssueCustomField":+ case "SingleGroupIssueCustomField":+ case "SingleVersionIssueCustomField":+ String fieldValue = customFieldNode.get("value").asText(null);+ if (fieldValue != null) {+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);+ if (mapped == null) {+ unmappedIssueFields.add(fieldName);+ extraIssueInfo.put(fieldName, fieldValue);+ } else if (!(mapped.getFirst() instanceof TextField)) {+ mismatchedIssueFields.put(fieldName, "Should be mapped to a text field");+ extraIssueInfo.put(fieldName, fieldValue);+ } else {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(mapped.getFirst().getName());+ issueField.setType(InputSpec.TEXT);+ issueField.setValue(fieldValue);+ issue.getFields().add(issueField);+ }+ }+ break;+ case "MultiBuildIssueCustomField":+ case "MultiGroupIssueCustomField":+ case "MultiVersionIssueCustomField":+ List<String> fieldValues = new ArrayList<>();+ for (JsonNode valueNode: customFieldNode.get("value"))+ fieldValues.add(valueNode.get("name").asText());+ if (!fieldValues.isEmpty()) {+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);+ if (mapped == null) {+ unmappedIssueFields.add(fieldName);+ extraIssueInfo.put(fieldName, joinAsMultilineHtml(fieldValues));+ } else if (!(mapped.getFirst() instanceof TextField)) {+ mismatchedIssueFields.put(fieldName, "Should be mapped to a text field");+ extraIssueInfo.put(fieldName, joinAsMultilineHtml(fieldValues));+ } else {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(mapped.getFirst().getName());+ issueField.setType(InputSpec.TEXT);+ issueField.setValue(StringUtils.join(fieldValues, ", "));+ issue.getFields().add(issueField);+ }+ }+ break;+ case "SingleEnumIssueCustomField":+ case "SingleOwnedIssueCustomField":+ if (customFieldNode.hasNonNull("value")) {+ fieldValue = customFieldNode.get("value").get("name").asText();+ String fieldNameWithValue = fieldName + "::" + fieldValue;+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldNameWithValue);+ if (mapped == null) {+ unmappedIssueFields.add(fieldNameWithValue);+ extraIssueInfo.put(fieldName, fieldValue);+ } else if (!(mapped.getFirst() instanceof ChoiceField) || mapped.getFirst().isAllowMultiple()) {+ mismatchedIssueFields.put(fieldNameWithValue, "Should be mapped to a single-valued enum field");+ extraIssueInfo.put(fieldName, fieldValue);+ } else {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(mapped.getFirst().getName());+ issueField.setType(InputSpec.ENUMERATION);+ issueField.setValue(mapped.getSecond());+ issue.getFields().add(issueField);+ }+ }+ break;+ case "MultiEnumIssueCustomField":+ case "MultiOwnedIssueCustomField":+ fieldValues = new ArrayList<>();+ for (JsonNode valueNode: customFieldNode.get("value"))+ fieldValues.add(valueNode.get("name").asText());++ boolean hasErrors = false;+ List<IssueField> issueFields = new ArrayList<>();+ for (String each: fieldValues) {+ String fieldNameWithValue = fieldName + "::" + each;+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldNameWithValue);+ if (mapped == null) {+ unmappedIssueFields.add(fieldNameWithValue);+ hasErrors = true;+ } else if (!(mapped.getFirst() instanceof ChoiceField) || !mapped.getFirst().isAllowMultiple()) {+ mismatchedIssueFields.put(fieldNameWithValue, "Should be mapped to a multi-valued enum field");+ hasErrors = true;+ } else {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(mapped.getFirst().getName());+ issueField.setType(InputSpec.ENUMERATION);+ issueField.setValue(mapped.getSecond());+ issueFields.add(issueField);+ }+ }+ if (hasErrors)+ extraIssueInfo.put(fieldName, joinAsMultilineHtml(fieldValues));+ else+ issue.getFields().addAll(issueFields);+ break;+ case "SingleUserIssueCustomField":+ if (customFieldNode.hasNonNull("value")) {+ JsonNode valueNode = customFieldNode.get("value");++ String login = valueNode.get("login").asText();+ String fullName = valueNode.get("name").asText();+ String email = getEmail(valueNode);++ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);+ if (mapped == null) {+ unmappedIssueFields.add(fieldName);+ extraIssueInfo.put(fieldName, fullName);+ } else if (!(mapped.getFirst() instanceof UserChoiceField) || mapped.getFirst().isAllowMultiple()) {+ mismatchedIssueFields.put(fieldName, "Should be mapped to a single-valued user field");+ extraIssueInfo.put(fieldName, fullName);+ } else {+ if (email != null) {+ User user = OneDev.getInstance(UserManager.class).findByVerifiedEmailAddress(email);+ if (user != null) {+ fieldValue = user.getName();+ } else {+ fieldValue = null;+ nonExistentLogins.add(login);+ }+ } else {+ fieldValue = null;+ nonExistentLogins.add(login);+ }++ if (fieldValue != null) {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(mapped.getFirst().getName());+ issueField.setType(InputSpec.USER);+ issueField.setValue(fieldValue);+ issue.getFields().add(issueField);+ }+ }+ }+ break;+ case "MultiUserIssueCustomField":+ List<String> fullNames = new ArrayList<>();+ for (JsonNode valueNode: customFieldNode.get("value"))+ fullNames.add(valueNode.get("name").asText());+ if (!fullNames.isEmpty()) {+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);+ if (mapped == null) {+ unmappedIssueFields.add(fieldName);+ extraIssueInfo.put(fieldName, joinAsMultilineHtml(fullNames));+ } else if (!(mapped.getFirst() instanceof UserChoiceField) || !mapped.getFirst().isAllowMultiple()) {+ mismatchedIssueFields.put(fieldName, "Should be mapped to a multi-valued user field");+ extraIssueInfo.put(fieldName, joinAsMultilineHtml(fullNames));+ } else {+ for (JsonNode valueNode: customFieldNode.get("value")) {+ String login = valueNode.get("login").asText();+ String email = getEmail(valueNode);++ if (email != null) {+ User user = OneDev.getInstance(UserManager.class).findByVerifiedEmailAddress(email);+ if (user != null) {+ fieldValue = user.getName();+ } else {+ fieldValue = null;+ nonExistentLogins.add(login);+ }+ } else {+ fieldValue = null;+ nonExistentLogins.add(login);+ }++ if (fieldValue != null) {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(mapped.getFirst().getName());+ issueField.setType(InputSpec.USER);+ issueField.setValue(fieldValue);+ issue.getFields().add(issueField);+ }+ }+ }+ }+ break;+ case "DateIssueCustomField":+ if (customFieldNode.hasNonNull("value")) {+ Date date = new Date(customFieldNode.get("value").asLong());+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);+ if (mapped == null) {+ unmappedIssueFields.add(fieldName);+ extraIssueInfo.put(fieldName, DateUtils.formatDate(date));+ } else if (!(mapped.getFirst() instanceof DateField)) {+ mismatchedIssueFields.put(fieldName, "Should be mapped to a date field");+ extraIssueInfo.put(fieldName, DateUtils.formatDate(date));+ } else {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(mapped.getFirst().getName());+ issueField.setType(InputSpec.DATE);+ issueField.setValue(DateUtils.formatDate(date));+ issue.getFields().add(issueField);+ }+ }+ break;+ case "SimpleIssueCustomField":+ if (customFieldNode.hasNonNull("value")) {+ JsonNode projectCustomField = customFieldNode.get("projectCustomField");+ JsonNode fieldNode = projectCustomField.get("field");+ if (fieldNode != null) {+ switch (fieldNode.get("fieldType").get("id").asText()) {+ case "date":+ Date date = new Date(customFieldNode.get("value").asLong());+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);+ if (mapped == null) {+ unmappedIssueFields.add(fieldName);+ extraIssueInfo.put(fieldName, DateUtils.formatDateTime(date));+ } else if (!(mapped.getFirst() instanceof DateTimeField)) {+ mismatchedIssueFields.put(fieldName, "Should be mapped to a datetime field");+ extraIssueInfo.put(fieldName, DateUtils.formatDateTime(date));+ } else {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(mapped.getFirst().getName());+ issueField.setType(InputSpec.DATE_TIME);+ issueField.setValue(DateUtils.formatDateTime(date));+ issue.getFields().add(issueField);+ }+ break;+ case "float":+ fieldValue = customFieldNode.get("value").asText();+ mapped = fieldMappings.get(fieldName);+ if (mapped == null) {+ unmappedIssueFields.add(fieldName);+ extraIssueInfo.put(fieldName, fieldValue);+ } else if (!(mapped.getFirst() instanceof FloatField)) {+ mismatchedIssueFields.put(fieldName, "Should be mapped to a float field");+ extraIssueInfo.put(fieldName, fieldValue);+ } else {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(mapped.getFirst().getName());+ issueField.setType(InputSpec.FLOAT);+ issueField.setValue(fieldValue);+ issue.getFields().add(issueField);+ }+ break;+ case "integer":+ fieldValue = customFieldNode.get("value").asText();+ mapped = fieldMappings.get(fieldName);+ if (mapped == null) {+ unmappedIssueFields.add(fieldName);+ extraIssueInfo.put(fieldName, fieldValue);+ } else if (!(mapped.getFirst() instanceof IntegerField)) {+ mismatchedIssueFields.put(fieldName, "Should be mapped to an integer field");+ extraIssueInfo.put(fieldName, fieldValue);+ } else {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(mapped.getFirst().getName());+ issueField.setType(InputSpec.INTEGER);+ issueField.setValue(fieldValue);+ issue.getFields().add(issueField);+ }+ break;+ case "string":+ fieldValue = customFieldNode.get("value").asText();+ mapped = fieldMappings.get(fieldName);+ if (mapped == null) {+ unmappedIssueFields.add(fieldName);+ extraIssueInfo.put(fieldName, fieldValue);+ } else if (!(mapped.getFirst() instanceof TextField)) {+ mismatchedIssueFields.put(fieldName, "Should be mapped to a text field");+ extraIssueInfo.put(fieldName, fieldValue);+ } else {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(mapped.getFirst().getName());+ issueField.setType(InputSpec.TEXT);+ issueField.setValue(fieldValue);+ issue.getFields().add(issueField);+ }+ break;+ }+ }+ }+ break;+ case "TextIssueCustomField":+ if (customFieldNode.hasNonNull("value")) {+ fieldValue = customFieldNode.get("value").get("text").asText();+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);+ if (mapped == null) {+ unmappedIssueFields.add(fieldName);+ extraIssueInfo.put(fieldName, fieldValue);+ } else if (!(mapped.getFirst() instanceof TextField)+ || !((TextField)mapped.getFirst()).isMultiline()) {+ mismatchedIssueFields.put(fieldName, "Should be mapped to a multi-line text field");+ extraIssueInfo.put(fieldName, fieldValue);+ } else {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(mapped.getFirst().getName());+ issueField.setType(InputSpec.TEXT);+ issueField.setValue(fieldValue);+ issue.getFields().add(issueField);+ }+ }+ break;+ case "PeriodIssueCustomField":+ if (customFieldNode.hasNonNull("value")) {+ fieldValue = customFieldNode.get("value").get("presentation").asText();+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);+ if (mapped == null) {+ unmappedIssueFields.add(fieldName);+ extraIssueInfo.put(fieldName, fieldValue);+ } else if (!(mapped.getFirst() instanceof WorkingPeriodField)) {+ mismatchedIssueFields.put(fieldName, "Should be mapped to a working period field");+ extraIssueInfo.put(fieldName, fieldValue);+ } else {+ IssueField issueField = new IssueField();+ issueField.setIssue(issue);+ issueField.setName(mapped.getFirst().getName());+ issueField.setType(InputSpec.WORKING_PERIOD);+ issueField.setValue(fieldValue);+ issue.getFields().add(issueField);+ }+ }+ break;+ }+ }+ if (issue.getState() == null)+ issue.setState(initialState.getName());++ for (JsonNode linkNode: issueNode.get("links")) {+ List<Long> linkedIssueNumbers = new ArrayList<>();+ for (JsonNode linkedIssueNode: linkNode.get("issues"))+ linkedIssueNumbers.add(linkedIssueNode.get("numberInProject").asLong());++ if (!linkedIssueNumbers.isEmpty() && linkNode.hasNonNull("linkType")) {+ JsonNode linkTypeNode = linkNode.get("linkType");+ String direction = linkNode.get("direction").asText();+ String linkName = linkTypeNode.get("name").asText(null);+ if (linkName != null) {+ LinkSpec linkSpec = linkMappings.get(linkName);+ if (linkSpec == null) {+ List<String> linkedIssueLinks = new ArrayList<>();+ for (Long issueNumber: linkedIssueNumbers)+ linkedIssueLinks.add(youTrackProjectShortName + "-" + issueNumber);++ unmappedIssueLinks.add(linkName);+ switch (direction) {+ case "BOTH":+ case "OUTWARD":+ linkName = linkTypeNode.get("sourceToTarget").asText(null);+ if (linkName != null)+ extraIssueInfo.put(linkName, joinAsMultilineHtml(linkedIssueLinks));+ break;+ case "INWARD":+ linkName = linkTypeNode.get("targetToSource").asText(null);+ if (linkName != null)+ extraIssueInfo.put(linkName, joinAsMultilineHtml(linkedIssueLinks));+ break;+ }+ } else if ("OUTWARD".equals(direction)) {+ issueLinkInfo.put(oldNumber, new Pair<>(linkSpec, linkedIssueNumbers));+ } else if ("BOTH".equals(direction)) {+ Set<Set<Long>> value = processedSymmetricLinks.get(linkSpec.getId());+ if (value == null) {+ value = new HashSet<>();+ processedSymmetricLinks.put(linkSpec.getId(), value);+ }+ List<Long> filteredIssueNumbers = new ArrayList<>();+ for (Long issueNumber: linkedIssueNumbers) {+ Set<Long> linkSides = Sets.newHashSet(oldNumber, issueNumber);+ if (value.add(linkSides))+ filteredIssueNumbers.add(issueNumber);+ }+ if (!filteredIssueNumbers.isEmpty())+ issueLinkInfo.put(oldNumber, new Pair<>(linkSpec, filteredIssueNumbers));+ }+ }+ }+ }++ List<String> currentUnmappedTags = new ArrayList<>();+ for (JsonNode tagNode: issueNode.get("tags")) {+ String tagName = tagNode.get("name").asText();+ Pair<FieldSpec, String> mapped = tagMappings.get(tagName);+ if (mapped != null) {+ IssueField tagField = new IssueField();+ tagField.setIssue(issue);+ tagField.setName(mapped.getFirst().getName());+ tagField.setType(InputSpec.ENUMERATION);+ tagField.setValue(mapped.getSecond());+ tagField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));+ issue.getFields().add(tagField);+ } else {+ currentUnmappedTags.add(HtmlEscape.escapeHtml5(tagName));+ unmappedIssueTags.add(HtmlEscape.escapeHtml5(tagName));+ }+ }+ if (!currentUnmappedTags.isEmpty())+ extraIssueInfo.put("Tags", joinAsMultilineHtml(currentUnmappedTags));++ for (JsonNode commentNode: issueNode.get("comments")) {+ String commentContent = commentNode.get("text").asText(null);+ if (commentContent != null || !commentNode.get("attachments").isEmpty()) {+ IssueComment comment = new IssueComment();+ comment.setIssue(issue);+ if (!dryRun) {+ List<JsonNode> attachmentNodes = new ArrayList<>();+ for (JsonNode attachmentNode: commentNode.get("attachments"))+ attachmentNodes.add(attachmentNode);+ comment.setContent(processAttachments(issue.getUUID(), readableId,+ commentContent, attachmentNodes, tooLargeAttachments));+ }+ comment.setDate(new Date(commentNode.get("created").asLong(System.currentTimeMillis())));+ if (commentNode.hasNonNull("author")) {+ JsonNode authorNode = commentNode.get("author");+ String email = getEmail(authorNode);+ String login = authorNode.get("login").asText();+ if (email != null) {+ User user = OneDev.getInstance(UserManager.class).findByVerifiedEmailAddress(email);+ if (user != null) {+ comment.setUser(user);+ } else {+ comment.setUser(OneDev.getInstance(UserManager.class).getUnknown());+ nonExistentLogins.add(login);+ }+ } else {+ comment.setUser(OneDev.getInstance(UserManager.class).getUnknown());+ nonExistentLogins.add(login);+ }+ } else {+ issue.setSubmitter(SecurityUtils.getUser());+ }+ issue.getComments().add(comment);+ }+ }++ issue.setCommentCount(issue.getComments().size());++ Set<String> fieldAndValues = new HashSet<>();+ for (IssueField field: issue.getFields()) {+ String fieldAndValue = field.getName() + "::" + field.getValue();+ if (!fieldAndValues.add(fieldAndValue)) {+ String errorMessage = String.format(+ "Duplicate issue field mapping (issue: %s, field: %s)",+ readableId, fieldAndValue);+ throw new ExplicitException(errorMessage);+ }+ }++ if (!extraIssueInfo.isEmpty()) {+ StringBuilder builder = new StringBuilder("|");+ for (String key: extraIssueInfo.keySet())+ builder.append(key).append("|");+ builder.append("\n|");+ extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));+ builder.append("\n|");+ for (String value: extraIssueInfo.values())+ builder.append(value).append("|");++ if (issue.getDescription() != null)+ issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());+ else+ issue.setDescription(builder.toString());+ }+ issues.add(issue);+ }+ logger.log("Imported " + numOfImportedIssues.addAndGet(pageData.size()) + " issues");+ }++ };++ apiEndpoint = getApiEndpoint("/admin/projects/" + youTrackProjectId + "/issues?fields=" + fields);+ list(client, apiEndpoint, pageDataConsumer, logger);++ if (!dryRun) {+ ReferenceMigrator migrator = new ReferenceMigrator(Issue.class, issueNumberMappings);+ Dao dao = OneDev.getInstance(Dao.class);+ for (Issue issue: issues) {+ if (issue.getDescription() != null)+ issue.setDescription(migrator.migratePrefixed(issue.getDescription(), youTrackProjectShortName + "-"));++ getIssueManager().save(issue);+ for (IssueSchedule schedule: issue.getSchedules())+ dao.persist(schedule);+ for (IssueField field: issue.getFields())+ dao.persist(field);+ for (IssueComment comment: issue.getComments()) {+ comment.setContent(migrator.migratePrefixed(comment.getContent(), youTrackProjectShortName + "-"));+ dao.persist(comment);+ }+ }++ for (Map.Entry<Long, Pair<LinkSpec, List<Long>>> entry: issueLinkInfo.entrySet()) {+ Issue source = issueMappings.get(entry.getKey());+ if (source != null) {+ for (Long targetNumber: entry.getValue().getSecond()) {+ Issue target = issueMappings.get(targetNumber);+ if (target != null) {+ IssueLink link = new IssueLink();+ link.setSource(source);+ link.setTarget(target);+ link.setSpec(entry.getValue().getFirst());+ OneDev.getInstance(IssueLinkManager.class).save(link);+ }+ }+ }+ }+ }++ ImportResult result = new ImportResult();+ result.mismatchedIssueFields.putAll(mismatchedIssueFields);+ result.nonExistentLogins.addAll(nonExistentLogins);+ result.tooLargeAttachments.addAll(tooLargeAttachments);+ result.unmappedIssueFields.addAll(unmappedIssueFields);+ result.unmappedIssueStates.addAll(unmappedIssueStates);+ result.unmappedIssueLinks.addAll(unmappedIssueLinks);+ result.unmappedIssueTags.addAll(unmappedIssueTags);++ return result;+ } finally {+ client.close();+ }+ }++ private IssueManager getIssueManager() {+ return OneDev.getInstance(IssueManager.class);+ }++ private GlobalIssueSetting getIssueSetting() {+ return OneDev.getInstance(SettingManager.class).getIssueSetting();+ }++ private List<JsonNode> list(Client client, String apiEndpoint, TaskLogger logger) {+ List<JsonNode> result = new ArrayList<>();+ list(client, apiEndpoint, new PageDataConsumer() {++ @Override+ public void consume(List<JsonNode> pageData) {+ result.addAll(pageData);+ }++ }, logger);+ return result;+ }++ String importProjects(ImportProjects projects, ImportOption option, boolean dryRun, TaskLogger logger) {+ Map<String, String> youTrackProjectIds = new HashMap<>();+ Map<String, String> youTrackProjectDescriptions = new HashMap<>();+ Collection<Long> projectIds = new ArrayList<>();++ Client client = newClient();+ try {+ String apiEndpoint = getApiEndpoint("/admin/projects?fields=id,name,description");+ for (JsonNode projectNode: list(client, apiEndpoint, logger)) {+ String projectName = projectNode.get("name").asText();+ youTrackProjectIds.put(projectName, projectNode.get("id").asText());+ youTrackProjectDescriptions.put(projectName, projectNode.get("description").asText(null));+ }++ ImportResult result = new ImportResult();++ for (ProjectMapping projectMapping: projects.getProjectMappings()) {+ logger.log("Importing project '" + projectMapping.getYouTrackProject() + "'...");++ String youTrackProjectId = youTrackProjectIds.get(projectMapping.getYouTrackProject());+ if (youTrackProjectId == null)+ throw new ExplicitException("Unable to find YouTrack project: " + projectMapping.getYouTrackProject());++ ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);+ Project project = projectManager.initialize(projectMapping.getOneDevProject());+ Preconditions.checkState(project.isNew());+ project.setDescription(youTrackProjectDescriptions.get(projectMapping.getYouTrackProject()));+ project.setIssueManagement(true);++ if (!dryRun) {+ projectManager.create(project);+ projectIds.add(project.getId());+ }++ ImportResult currentResult = importIssues(youTrackProjectId, project,+ true, option, dryRun, logger);+ result.mismatchedIssueFields.putAll(currentResult.mismatchedIssueFields);+ result.nonExistentLogins.addAll(currentResult.nonExistentLogins);+ result.tooLargeAttachments.addAll(currentResult.tooLargeAttachments);+ result.unmappedIssueFields.addAll(currentResult.unmappedIssueFields);+ result.unmappedIssueStates.addAll(currentResult.unmappedIssueStates);+ result.unmappedIssueLinks.addAll(currentResult.unmappedIssueLinks);+ result.unmappedIssueTags.addAll(currentResult.unmappedIssueTags);+ }++ return result.toHtml("Projects imported successfully");+ } catch (Exception e) {+ for (Long projectId: projectIds)+ OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);+ throw ExceptionUtils.unchecked(e);+ } finally {+ client.close();+ }+ }++ private void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer, TaskLogger logger) {+ URI uri;+ try {+ uri = new URIBuilder(apiEndpoint)+ .addParameter("$top", String.valueOf(PER_PAGE)).build();+ } catch (URISyntaxException e) {+ throw new RuntimeException(e);+ }++ int page = 0;+ while (true) {+ try {+ URIBuilder builder = new URIBuilder(uri);+ builder.addParameter("$skip", String.valueOf(page*PER_PAGE));+ List<JsonNode> pageData = new ArrayList<>();+ for (JsonNode each: JerseyUtils.get(client, builder.build().toString(), logger))+ pageData.add(each);+ pageDataConsumer.consume(pageData);+ if (pageData.size() < PER_PAGE)+ break;+ page++;+ } catch (URISyntaxException|InterruptedException e) {+ throw new RuntimeException(e);+ }+ }+ }++ String importIssues(Project project, String youTrackProject, ImportOption option,+ boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {+ logger.log("Importing issues from '" + youTrackProject + "'...");+ Client client = newClient();+ try {+ String apiEndpoint = getApiEndpoint("/admin/projects?fields=id,name");+ for (JsonNode projectNode: list(client, apiEndpoint, logger)) {+ if (youTrackProject.equals(projectNode.get("name").asText())) {+ ImportResult result = importIssues(projectNode.get("id").asText(),+ project, retainIssueNumbers, option, dryRun, logger);+ return result.toHtml("Issues imported successfully");+ }+ }+ throw new ExplicitException("Unable to find YouTrack project: " + youTrackProject);+ } finally {+ client.close();+ }+ }+@Overridepublic boolean isValid(ConstraintValidatorContext context) {Client client = ClientBuilder.newClient();@@ -136,34 +1215,4 @@ public class ImportServer implements Serializable, Validatable {return true;}- List<String> getProjectChoices() {- List<String> choices = new ArrayList<>();-- Client client = newClient();- try {- TaskLogger logger = new TaskLogger() {-- @Override- public void log(String message, String sessionId) {- ImportServer.logger.info(message);- }-- };- String apiEndpoint = getApiEndpoint("/admin/projects?fields=name");- for (JsonNode projectNode: ImportUtils.list(client, apiEndpoint, logger))- choices.add(projectNode.get("name").asText());- } finally {- client.close();- }-- Collections.sort(choices);- return choices;- }-- Client newClient() {- Client client = ClientBuilder.newClient();- client.register(HttpAuthenticationFeature.basic(getUserName(), getPassword()));- return client;- }-}diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportUtils.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportUtils.javadeleted file mode 100644index bdab8915c0..0000000000--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportUtils.java+++ /dev/null@@ -1,935 +0,0 @@-package io.onedev.server.plugin.imports.youtrack;--import java.io.IOException;-import java.io.InputStream;-import java.net.URI;-import java.net.URISyntaxException;-import java.util.ArrayList;-import java.util.Collection;-import java.util.Date;-import java.util.HashMap;-import java.util.HashSet;-import java.util.LinkedHashMap;-import java.util.LinkedHashSet;-import java.util.List;-import java.util.Map;-import java.util.Set;-import java.util.concurrent.atomic.AtomicInteger;--import javax.annotation.Nullable;-import javax.ws.rs.client.Client;-import javax.ws.rs.client.Invocation;-import javax.ws.rs.client.WebTarget;-import javax.ws.rs.core.Response;--import org.apache.commons.lang3.StringUtils;-import org.apache.http.client.utils.URIBuilder;-import org.unbescape.html.HtmlEscape;--import com.fasterxml.jackson.databind.JsonNode;-import com.google.common.collect.Sets;--import io.onedev.commons.utils.ExplicitException;-import io.onedev.commons.utils.TaskLogger;-import io.onedev.server.OneDev;-import io.onedev.server.entitymanager.IssueLinkManager;-import io.onedev.server.entitymanager.IssueManager;-import io.onedev.server.entitymanager.LinkSpecManager;-import io.onedev.server.entitymanager.SettingManager;-import io.onedev.server.entitymanager.UserManager;-import io.onedev.server.entityreference.ReferenceMigrator;-import io.onedev.server.model.Issue;-import io.onedev.server.model.IssueComment;-import io.onedev.server.model.IssueField;-import io.onedev.server.model.IssueLink;-import io.onedev.server.model.IssueSchedule;-import io.onedev.server.model.LinkSpec;-import io.onedev.server.model.Project;-import io.onedev.server.model.User;-import io.onedev.server.model.support.LastUpdate;-import io.onedev.server.model.support.administration.GlobalIssueSetting;-import io.onedev.server.model.support.inputspec.InputSpec;-import io.onedev.server.model.support.issue.StateSpec;-import io.onedev.server.model.support.issue.field.spec.ChoiceField;-import io.onedev.server.model.support.issue.field.spec.DateField;-import io.onedev.server.model.support.issue.field.spec.DateTimeField;-import io.onedev.server.model.support.issue.field.spec.FieldSpec;-import io.onedev.server.model.support.issue.field.spec.FloatField;-import io.onedev.server.model.support.issue.field.spec.IntegerField;-import io.onedev.server.model.support.issue.field.spec.TextField;-import io.onedev.server.model.support.issue.field.spec.UserChoiceField;-import io.onedev.server.model.support.issue.field.spec.WorkingPeriodField;-import io.onedev.server.persistence.dao.Dao;-import io.onedev.server.security.SecurityUtils;-import io.onedev.server.util.DateUtils;-import io.onedev.server.util.JerseyUtils;-import io.onedev.server.util.JerseyUtils.PageDataConsumer;-import io.onedev.server.util.Pair;--public class ImportUtils {-- static final String NAME = "YouTrack";-- static final int PER_PAGE = 50;-- static ImportOption buildImportOption(ImportServer server, Collection<JsonNode> projectNodes, TaskLogger logger) {- ImportOption importOption = new ImportOption();- Client client = server.newClient();- try {- Set<String> youTrackIssueStates = new LinkedHashSet<>();- Set<String> youTrackIssueFields = new LinkedHashSet<>();-- for (JsonNode projectNode: projectNodes) {- for (JsonNode customFieldNode: projectNode.get("customFields")) {- String fieldName = customFieldNode.get("field").get("name").asText();- JsonNode bundleNode = customFieldNode.get("bundle");- if (bundleNode != null) {- String bundleType = bundleNode.get("$type").asText();- if (bundleType.equals("StateBundle")) {- for (JsonNode valueNode: bundleNode.get("values"))- youTrackIssueStates.add(valueNode.get("name").asText());- } else if (bundleType.equals("EnumBundle") || bundleType.equals("OwnedBundle")) {- for (JsonNode valueNode: bundleNode.get("values"))- youTrackIssueFields.add(fieldName + "::" + valueNode.get("name").asText());- } else {- youTrackIssueFields.add(fieldName);- }- } else {- youTrackIssueFields.add(fieldName);- }- }- }-- for (String youTrackIssueState: youTrackIssueStates) {- IssueStateMapping mapping = new IssueStateMapping();- mapping.setYouTrackIssueState(youTrackIssueState);- importOption.getIssueStateMappings().add(mapping);- }-- for (String youTrackIssueField: youTrackIssueFields) {- IssueFieldMapping mapping = new IssueFieldMapping();- mapping.setYouTrackIssueField(youTrackIssueField);- importOption.getIssueFieldMappings().add(mapping);- }-- String apiEndpoint = server.getApiEndpoint("/issueTags?fields=name");- for (JsonNode tagNode: list(client, apiEndpoint, logger)) {- IssueTagMapping mapping = new IssueTagMapping();- mapping.setYouTrackIssueTag(tagNode.get("name").asText());- importOption.getIssueTagMappings().add(mapping);- }-- apiEndpoint = server.getApiEndpoint("/issueLinkTypes?fields=name");- for (JsonNode tagNode: list(client, apiEndpoint, logger)) {- IssueLinkMapping mapping = new IssueLinkMapping();- mapping.setYouTrackIssueLink(tagNode.get("name").asText());- importOption.getIssueLinkMappings().add(mapping);- }-- } finally {- client.close();- }- return importOption;- }-- static ImportResult importIssues(ImportServer server, String youTrackProjectId,- Project oneDevProject, boolean retainIssueNumbers, ImportOption importOption, boolean dryRun, TaskLogger logger) {- Client client = server.newClient();- try {- String apiEndpoint = server.getApiEndpoint("/admin/projects/" + youTrackProjectId + "?fields=shortName");- String youTrackProjectShortName = JerseyUtils.get(client, apiEndpoint, logger).get("shortName").asText();-- Map<String, String> stateMappings = new HashMap<>();- Map<String, Pair<FieldSpec, String>> fieldMappings = new HashMap<>();- Map<String, Pair<FieldSpec, String>> tagMappings = new HashMap<>();- Map<String, LinkSpec> linkMappings = new HashMap<>();-- Map<Long, Long> issueNumberMappings = new HashMap<>();- Map<Long, Issue> issueMappings = new HashMap<>();-- for (IssueStateMapping mapping: importOption.getIssueStateMappings())- stateMappings.put(mapping.getYouTrackIssueState(), mapping.getOneDevIssueState());-- for (IssueFieldMapping mapping: importOption.getIssueFieldMappings()) {- String oneDevFieldName;- String oneDevFieldValue;- if (mapping.getOneDevIssueField().contains("::")) {- oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");- oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");- } else {- oneDevFieldName = mapping.getOneDevIssueField();- oneDevFieldValue = null;- }- FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);- if (fieldSpec == null)- throw new ExplicitException("No field spec found: " + oneDevFieldName);- fieldMappings.put(mapping.getYouTrackIssueField(), new Pair<>(fieldSpec, oneDevFieldValue));- }- for (IssueTagMapping mapping: importOption.getIssueTagMappings()) {- String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");- String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");- FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);- if (fieldSpec == null)- throw new ExplicitException("No field spec found: " + oneDevFieldName);- tagMappings.put(mapping.getYouTrackIssueTag(), new Pair<>(fieldSpec, oneDevFieldValue));- }- for (IssueLinkMapping mapping: importOption.getIssueLinkMappings()) {- LinkSpec linkSpec = OneDev.getInstance(LinkSpecManager.class).find(mapping.getOneDevIssueLink());- if (linkSpec == null)- throw new ExplicitException("No link spec found: " + mapping.getOneDevIssueLink());- linkMappings.put(mapping.getYouTrackIssueLink(), linkSpec);- }-- Set<String> nonExistentLogins = new LinkedHashSet<>();- Set<String> unmappedIssueTags = new LinkedHashSet<>();- Set<String> unmappedIssueFields = new LinkedHashSet<>();- Set<String> unmappedIssueLinks = new LinkedHashSet<>();- Set<String> unmappedIssueStates = new LinkedHashSet<>();- Map<String, String> mismatchedIssueFields = new LinkedHashMap<>();- Set<String> tooLargeAttachments = new LinkedHashSet<>();-- AtomicInteger numOfImportedIssues = new AtomicInteger(0);-- List<Issue> issues = new ArrayList<>();- Map<Long, Pair<LinkSpec, List<Long>>> issueLinkInfo = new HashMap<>();- Map<Long, Set<Set<Long>>> processedSymmetricLinks = new HashMap<>();-- String fields = ""- + "idReadable,"- + "numberInProject,"- + "summary,"- + "description,"- + "created,"- + "comments(created,text,author(login,name,email),attachments(name,url,size)),"- + "attachments(name,url,comment,size),"- + "reporter(login,name,email),"- + "tags(name),"- + "customFields(name,value(name,login,email,presentation,text),projectCustomField(field(fieldType(id)))),"- + "links(direction,linkType(name,sourceToTarget,targetToSource),issues(numberInProject))";- PageDataConsumer pageDataConsumer = new PageDataConsumer() {-- @Nullable- private String processAttachments(String issueUUID, String readableIssueId, @Nullable String markdown,- List<JsonNode> attachmentNodes, Set<String> tooLargeAttachments) {- if (markdown == null)- markdown = "";-- Map<String, String> unreferencedAttachments = new LinkedHashMap<>();-- long maxUploadFileSize = OneDev.getInstance(SettingManager.class)- .getPerformanceSetting().getMaxUploadFileSize()*1L*1024*1024;- for (JsonNode attachmentNode: attachmentNodes) {- String attachmentName = attachmentNode.get("name").asText(null);- String attachmentUrl = attachmentNode.get("url").asText(null);- long attachmentSize = attachmentNode.get("size").asLong(0);- if (attachmentSize != 0 && attachmentName != null && attachmentUrl != null) {- if (attachmentSize > maxUploadFileSize) {- tooLargeAttachments.add(readableIssueId + ":" + attachmentName);- } else {- if (!attachmentUrl.startsWith("/api"))- throw new ExplicitException("Unexpected attachment url: " + attachmentUrl);- attachmentUrl = attachmentUrl.substring("/api".length());-- String endpoint = server.getApiEndpoint(attachmentUrl);- WebTarget target = client.target(endpoint);- Invocation.Builder builder = target.request();- try (Response response = builder.get()) {- String errorMessage = JerseyUtils.checkStatus(endpoint, response);- if (errorMessage != null) {- throw new ExplicitException(String.format(- "Error downloading attachment (url: %s, error message: %s)",- endpoint, errorMessage));- }- try (InputStream is = response.readEntity(InputStream.class)) {- String oneDevAttachmentName = oneDevProject.saveAttachment(issueUUID, attachmentName, is);- String oneDevAttachmentUrl = oneDevProject.getAttachmentUrlPath(issueUUID, oneDevAttachmentName);- if (markdown.contains("(" + attachmentName + ")")) {- markdown = markdown.replace("(" + attachmentName + ")", "(" + oneDevAttachmentUrl + ")");- } else {- unreferencedAttachments.put(attachmentName, oneDevAttachmentUrl);- }- } catch (IOException e) {- throw new RuntimeException(e);- }- }- }- }- }-- if (!unreferencedAttachments.isEmpty()) {- markdown += "\n\n";- for (Map.Entry<String, String> entry: unreferencedAttachments.entrySet())- markdown += "[" + entry.getKey() + "](" + entry.getValue() + ") ";- }-- if (markdown.length() == 0)- markdown = null;-- return markdown;- }-- private String joinAsMultilineHtml(List<String> values) {- List<String> escapedValues = new ArrayList<>();- for (String value: values)- escapedValues.add(HtmlEscape.escapeHtml5(value));- return StringUtils.join(escapedValues, "<br>");- }-- @Nullable- private String getEmail(JsonNode userNode) {- JsonNode emailNode = userNode.get("email");- if (emailNode != null)- return emailNode.asText(null);- else- return null;- }-- @Override- public void consume(List<JsonNode> pageData) throws InterruptedException {- for (JsonNode issueNode: pageData) {- if (Thread.interrupted())- throw new InterruptedException();-- Issue issue = new Issue();- String readableId = issueNode.get("idReadable").asText();- Long oldNumber = issueNode.get("numberInProject").asLong();- Long newNumber;- if (dryRun || retainIssueNumbers)- newNumber = oldNumber;- else- newNumber = getIssueManager().getNextNumber(oneDevProject);- issue.setNumber(newNumber);- issueNumberMappings.put(oldNumber, newNumber);- issueMappings.put(oldNumber, issue);- issue.setTitle(issueNode.get("summary").asText());- issue.setDescription(issueNode.get("description").asText(null));- issue.setSubmitDate(new Date(issueNode.get("created").asLong(System.currentTimeMillis())));- issue.setProject(oneDevProject);- issue.setNumberScope(oneDevProject.getForkRoot());-- if (!dryRun) {- List<JsonNode> attachmentNodes = new ArrayList<>();- for (JsonNode attachmentNode: issueNode.get("attachments")) {- if (!attachmentNode.hasNonNull("comment"))- attachmentNodes.add(attachmentNode);- }- issue.setDescription(processAttachments(issue.getUUID(), readableId,- issue.getDescription(), attachmentNodes, tooLargeAttachments));- }-- Map<String, String> extraIssueInfo = new LinkedHashMap<>();-- if (issueNode.hasNonNull("reporter")) {- JsonNode reporterNode = issueNode.get("reporter");- String email = getEmail(reporterNode);- String login = reporterNode.get("login").asText();- if (email != null) {- User user = OneDev.getInstance(UserManager.class).findByVerifiedEmailAddress(email);- if (user != null) {- issue.setSubmitter(user);- } else {- issue.setSubmitter(OneDev.getInstance(UserManager.class).getUnknown());- nonExistentLogins.add(login);- }- } else {- issue.setSubmitter(OneDev.getInstance(UserManager.class).getUnknown());- nonExistentLogins.add(login);- }- } else {- issue.setSubmitter(SecurityUtils.getUser());- }-- LastUpdate lastUpdate = new LastUpdate();- lastUpdate.setActivity("Opened");- lastUpdate.setDate(issue.getSubmitDate());- lastUpdate.setUser(issue.getSubmitter());- issue.setLastUpdate(lastUpdate);-- StateSpec initialState = getIssueSetting().getInitialStateSpec();- for (JsonNode customFieldNode: issueNode.get("customFields")) {- String fieldName = customFieldNode.get("name").asText();- String fieldType = customFieldNode.get("$type").asText();- switch (fieldType) {- case "StateIssueCustomField":- if (customFieldNode.hasNonNull("value")) {- String state = customFieldNode.get("value").get("name").asText();- String mappedState = stateMappings.get(state);- if (mappedState != null)- issue.setState(mappedState);- else- unmappedIssueStates.add(state);- }- break;- case "SingleBuildIssueCustomField":- case "SingleGroupIssueCustomField":- case "SingleVersionIssueCustomField":- String fieldValue = customFieldNode.get("value").asText(null);- if (fieldValue != null) {- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);- if (mapped == null) {- unmappedIssueFields.add(fieldName);- extraIssueInfo.put(fieldName, fieldValue);- } else if (!(mapped.getFirst() instanceof TextField)) {- mismatchedIssueFields.put(fieldName, "Should be mapped to a text field");- extraIssueInfo.put(fieldName, fieldValue);- } else {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(mapped.getFirst().getName());- issueField.setType(InputSpec.TEXT);- issueField.setValue(fieldValue);- issue.getFields().add(issueField);- }- }- break;- case "MultiBuildIssueCustomField":- case "MultiGroupIssueCustomField":- case "MultiVersionIssueCustomField":- List<String> fieldValues = new ArrayList<>();- for (JsonNode valueNode: customFieldNode.get("value"))- fieldValues.add(valueNode.get("name").asText());- if (!fieldValues.isEmpty()) {- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);- if (mapped == null) {- unmappedIssueFields.add(fieldName);- extraIssueInfo.put(fieldName, joinAsMultilineHtml(fieldValues));- } else if (!(mapped.getFirst() instanceof TextField)) {- mismatchedIssueFields.put(fieldName, "Should be mapped to a text field");- extraIssueInfo.put(fieldName, joinAsMultilineHtml(fieldValues));- } else {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(mapped.getFirst().getName());- issueField.setType(InputSpec.TEXT);- issueField.setValue(StringUtils.join(fieldValues, ", "));- issue.getFields().add(issueField);- }- }- break;- case "SingleEnumIssueCustomField":- case "SingleOwnedIssueCustomField":- if (customFieldNode.hasNonNull("value")) {- fieldValue = customFieldNode.get("value").get("name").asText();- String fieldNameWithValue = fieldName + "::" + fieldValue;- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldNameWithValue);- if (mapped == null) {- unmappedIssueFields.add(fieldNameWithValue);- extraIssueInfo.put(fieldName, fieldValue);- } else if (!(mapped.getFirst() instanceof ChoiceField) || mapped.getFirst().isAllowMultiple()) {- mismatchedIssueFields.put(fieldNameWithValue, "Should be mapped to a single-valued enum field");- extraIssueInfo.put(fieldName, fieldValue);- } else {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(mapped.getFirst().getName());- issueField.setType(InputSpec.ENUMERATION);- issueField.setValue(mapped.getSecond());- issue.getFields().add(issueField);- }- }- break;- case "MultiEnumIssueCustomField":- case "MultiOwnedIssueCustomField":- fieldValues = new ArrayList<>();- for (JsonNode valueNode: customFieldNode.get("value"))- fieldValues.add(valueNode.get("name").asText());-- boolean hasErrors = false;- List<IssueField> issueFields = new ArrayList<>();- for (String each: fieldValues) {- String fieldNameWithValue = fieldName + "::" + each;- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldNameWithValue);- if (mapped == null) {- unmappedIssueFields.add(fieldNameWithValue);- hasErrors = true;- } else if (!(mapped.getFirst() instanceof ChoiceField) || !mapped.getFirst().isAllowMultiple()) {- mismatchedIssueFields.put(fieldNameWithValue, "Should be mapped to a multi-valued enum field");- hasErrors = true;- } else {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(mapped.getFirst().getName());- issueField.setType(InputSpec.ENUMERATION);- issueField.setValue(mapped.getSecond());- issueFields.add(issueField);- }- }- if (hasErrors)- extraIssueInfo.put(fieldName, joinAsMultilineHtml(fieldValues));- else- issue.getFields().addAll(issueFields);- break;- case "SingleUserIssueCustomField":- if (customFieldNode.hasNonNull("value")) {- JsonNode valueNode = customFieldNode.get("value");-- String login = valueNode.get("login").asText();- String fullName = valueNode.get("name").asText();- String email = getEmail(valueNode);-- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);- if (mapped == null) {- unmappedIssueFields.add(fieldName);- extraIssueInfo.put(fieldName, fullName);- } else if (!(mapped.getFirst() instanceof UserChoiceField) || mapped.getFirst().isAllowMultiple()) {- mismatchedIssueFields.put(fieldName, "Should be mapped to a single-valued user field");- extraIssueInfo.put(fieldName, fullName);- } else {- if (email != null) {- User user = OneDev.getInstance(UserManager.class).findByVerifiedEmailAddress(email);- if (user != null) {- fieldValue = user.getName();- } else {- fieldValue = null;- nonExistentLogins.add(login);- }- } else {- fieldValue = null;- nonExistentLogins.add(login);- }-- if (fieldValue != null) {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(mapped.getFirst().getName());- issueField.setType(InputSpec.USER);- issueField.setValue(fieldValue);- issue.getFields().add(issueField);- }- }- }- break;- case "MultiUserIssueCustomField":- List<String> fullNames = new ArrayList<>();- for (JsonNode valueNode: customFieldNode.get("value"))- fullNames.add(valueNode.get("name").asText());- if (!fullNames.isEmpty()) {- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);- if (mapped == null) {- unmappedIssueFields.add(fieldName);- extraIssueInfo.put(fieldName, joinAsMultilineHtml(fullNames));- } else if (!(mapped.getFirst() instanceof UserChoiceField) || !mapped.getFirst().isAllowMultiple()) {- mismatchedIssueFields.put(fieldName, "Should be mapped to a multi-valued user field");- extraIssueInfo.put(fieldName, joinAsMultilineHtml(fullNames));- } else {- for (JsonNode valueNode: customFieldNode.get("value")) {- String login = valueNode.get("login").asText();- String email = getEmail(valueNode);-- if (email != null) {- User user = OneDev.getInstance(UserManager.class).findByVerifiedEmailAddress(email);- if (user != null) {- fieldValue = user.getName();- } else {- fieldValue = null;- nonExistentLogins.add(login);- }- } else {- fieldValue = null;- nonExistentLogins.add(login);- }-- if (fieldValue != null) {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(mapped.getFirst().getName());- issueField.setType(InputSpec.USER);- issueField.setValue(fieldValue);- issue.getFields().add(issueField);- }- }- }- }- break;- case "DateIssueCustomField":- if (customFieldNode.hasNonNull("value")) {- Date date = new Date(customFieldNode.get("value").asLong());- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);- if (mapped == null) {- unmappedIssueFields.add(fieldName);- extraIssueInfo.put(fieldName, DateUtils.formatDate(date));- } else if (!(mapped.getFirst() instanceof DateField)) {- mismatchedIssueFields.put(fieldName, "Should be mapped to a date field");- extraIssueInfo.put(fieldName, DateUtils.formatDate(date));- } else {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(mapped.getFirst().getName());- issueField.setType(InputSpec.DATE);- issueField.setValue(DateUtils.formatDate(date));- issue.getFields().add(issueField);- }- }- break;- case "SimpleIssueCustomField":- if (customFieldNode.hasNonNull("value")) {- JsonNode projectCustomField = customFieldNode.get("projectCustomField");- JsonNode fieldNode = projectCustomField.get("field");- if (fieldNode != null) {- switch (fieldNode.get("fieldType").get("id").asText()) {- case "date":- Date date = new Date(customFieldNode.get("value").asLong());- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);- if (mapped == null) {- unmappedIssueFields.add(fieldName);- extraIssueInfo.put(fieldName, DateUtils.formatDateTime(date));- } else if (!(mapped.getFirst() instanceof DateTimeField)) {- mismatchedIssueFields.put(fieldName, "Should be mapped to a datetime field");- extraIssueInfo.put(fieldName, DateUtils.formatDateTime(date));- } else {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(mapped.getFirst().getName());- issueField.setType(InputSpec.DATE_TIME);- issueField.setValue(DateUtils.formatDateTime(date));- issue.getFields().add(issueField);- }- break;- case "float":- fieldValue = customFieldNode.get("value").asText();- mapped = fieldMappings.get(fieldName);- if (mapped == null) {- unmappedIssueFields.add(fieldName);- extraIssueInfo.put(fieldName, fieldValue);- } else if (!(mapped.getFirst() instanceof FloatField)) {- mismatchedIssueFields.put(fieldName, "Should be mapped to a float field");- extraIssueInfo.put(fieldName, fieldValue);- } else {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(mapped.getFirst().getName());- issueField.setType(InputSpec.FLOAT);- issueField.setValue(fieldValue);- issue.getFields().add(issueField);- }- break;- case "integer":- fieldValue = customFieldNode.get("value").asText();- mapped = fieldMappings.get(fieldName);- if (mapped == null) {- unmappedIssueFields.add(fieldName);- extraIssueInfo.put(fieldName, fieldValue);- } else if (!(mapped.getFirst() instanceof IntegerField)) {- mismatchedIssueFields.put(fieldName, "Should be mapped to an integer field");- extraIssueInfo.put(fieldName, fieldValue);- } else {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(mapped.getFirst().getName());- issueField.setType(InputSpec.INTEGER);- issueField.setValue(fieldValue);- issue.getFields().add(issueField);- }- break;- case "string":- fieldValue = customFieldNode.get("value").asText();- mapped = fieldMappings.get(fieldName);- if (mapped == null) {- unmappedIssueFields.add(fieldName);- extraIssueInfo.put(fieldName, fieldValue);- } else if (!(mapped.getFirst() instanceof TextField)) {- mismatchedIssueFields.put(fieldName, "Should be mapped to a text field");- extraIssueInfo.put(fieldName, fieldValue);- } else {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(mapped.getFirst().getName());- issueField.setType(InputSpec.TEXT);- issueField.setValue(fieldValue);- issue.getFields().add(issueField);- }- break;- }- }- }- break;- case "TextIssueCustomField":- if (customFieldNode.hasNonNull("value")) {- fieldValue = customFieldNode.get("value").get("text").asText();- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);- if (mapped == null) {- unmappedIssueFields.add(fieldName);- extraIssueInfo.put(fieldName, fieldValue);- } else if (!(mapped.getFirst() instanceof TextField)- || !((TextField)mapped.getFirst()).isMultiline()) {- mismatchedIssueFields.put(fieldName, "Should be mapped to a multi-line text field");- extraIssueInfo.put(fieldName, fieldValue);- } else {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(mapped.getFirst().getName());- issueField.setType(InputSpec.TEXT);- issueField.setValue(fieldValue);- issue.getFields().add(issueField);- }- }- break;- case "PeriodIssueCustomField":- if (customFieldNode.hasNonNull("value")) {- fieldValue = customFieldNode.get("value").get("presentation").asText();- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);- if (mapped == null) {- unmappedIssueFields.add(fieldName);- extraIssueInfo.put(fieldName, fieldValue);- } else if (!(mapped.getFirst() instanceof WorkingPeriodField)) {- mismatchedIssueFields.put(fieldName, "Should be mapped to a working period field");- extraIssueInfo.put(fieldName, fieldValue);- } else {- IssueField issueField = new IssueField();- issueField.setIssue(issue);- issueField.setName(mapped.getFirst().getName());- issueField.setType(InputSpec.WORKING_PERIOD);- issueField.setValue(fieldValue);- issue.getFields().add(issueField);- }- }- break;- }- }- if (issue.getState() == null)- issue.setState(initialState.getName());-- for (JsonNode linkNode: issueNode.get("links")) {- List<Long> linkedIssueNumbers = new ArrayList<>();- for (JsonNode linkedIssueNode: linkNode.get("issues"))- linkedIssueNumbers.add(linkedIssueNode.get("numberInProject").asLong());-- if (!linkedIssueNumbers.isEmpty() && linkNode.hasNonNull("linkType")) {- JsonNode linkTypeNode = linkNode.get("linkType");- String direction = linkNode.get("direction").asText();- String linkName = linkTypeNode.get("name").asText(null);- if (linkName != null) {- LinkSpec linkSpec = linkMappings.get(linkName);- if (linkSpec == null) {- List<String> linkedIssueLinks = new ArrayList<>();- for (Long issueNumber: linkedIssueNumbers)- linkedIssueLinks.add(youTrackProjectShortName + "-" + issueNumber);-- unmappedIssueLinks.add(linkName);- switch (direction) {- case "BOTH":- case "OUTWARD":- linkName = linkTypeNode.get("sourceToTarget").asText(null);- if (linkName != null)- extraIssueInfo.put(linkName, joinAsMultilineHtml(linkedIssueLinks));- break;- case "INWARD":- linkName = linkTypeNode.get("targetToSource").asText(null);- if (linkName != null)- extraIssueInfo.put(linkName, joinAsMultilineHtml(linkedIssueLinks));- break;- }- } else if ("OUTWARD".equals(direction)) {- issueLinkInfo.put(oldNumber, new Pair<>(linkSpec, linkedIssueNumbers));- } else if ("BOTH".equals(direction)) {- Set<Set<Long>> value = processedSymmetricLinks.get(linkSpec.getId());- if (value == null) {- value = new HashSet<>();- processedSymmetricLinks.put(linkSpec.getId(), value);- }- List<Long> filteredIssueNumbers = new ArrayList<>();- for (Long issueNumber: linkedIssueNumbers) {- Set<Long> linkSides = Sets.newHashSet(oldNumber, issueNumber);- if (value.add(linkSides))- filteredIssueNumbers.add(issueNumber);- }- if (!filteredIssueNumbers.isEmpty())- issueLinkInfo.put(oldNumber, new Pair<>(linkSpec, filteredIssueNumbers));- }- }- }- }-- List<String> currentUnmappedTags = new ArrayList<>();- for (JsonNode tagNode: issueNode.get("tags")) {- String tagName = tagNode.get("name").asText();- Pair<FieldSpec, String> mapped = tagMappings.get(tagName);- if (mapped != null) {- IssueField tagField = new IssueField();- tagField.setIssue(issue);- tagField.setName(mapped.getFirst().getName());- tagField.setType(InputSpec.ENUMERATION);- tagField.setValue(mapped.getSecond());- tagField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));- issue.getFields().add(tagField);- } else {- currentUnmappedTags.add(HtmlEscape.escapeHtml5(tagName));- unmappedIssueTags.add(HtmlEscape.escapeHtml5(tagName));- }- }- if (!currentUnmappedTags.isEmpty())- extraIssueInfo.put("Tags", joinAsMultilineHtml(currentUnmappedTags));-- for (JsonNode commentNode: issueNode.get("comments")) {- String commentContent = commentNode.get("text").asText(null);- if (commentContent != null || !commentNode.get("attachments").isEmpty()) {- IssueComment comment = new IssueComment();- comment.setIssue(issue);- if (!dryRun) {- List<JsonNode> attachmentNodes = new ArrayList<>();- for (JsonNode attachmentNode: commentNode.get("attachments"))- attachmentNodes.add(attachmentNode);- comment.setContent(processAttachments(issue.getUUID(), readableId,- commentContent, attachmentNodes, tooLargeAttachments));- }- comment.setDate(new Date(commentNode.get("created").asLong(System.currentTimeMillis())));- if (commentNode.hasNonNull("author")) {- JsonNode authorNode = commentNode.get("author");- String email = getEmail(authorNode);- String login = authorNode.get("login").asText();- if (email != null) {- User user = OneDev.getInstance(UserManager.class).findByVerifiedEmailAddress(email);- if (user != null) {- comment.setUser(user);- } else {- comment.setUser(OneDev.getInstance(UserManager.class).getUnknown());- nonExistentLogins.add(login);- }- } else {- comment.setUser(OneDev.getInstance(UserManager.class).getUnknown());- nonExistentLogins.add(login);- }- } else {- issue.setSubmitter(SecurityUtils.getUser());- }- issue.getComments().add(comment);- }- }-- issue.setCommentCount(issue.getComments().size());-- Set<String> fieldAndValues = new HashSet<>();- for (IssueField field: issue.getFields()) {- String fieldAndValue = field.getName() + "::" + field.getValue();- if (!fieldAndValues.add(fieldAndValue)) {- String errorMessage = String.format(- "Duplicate issue field mapping (issue: %s, field: %s)",- readableId, fieldAndValue);- throw new ExplicitException(errorMessage);- }- }-- if (!extraIssueInfo.isEmpty()) {- StringBuilder builder = new StringBuilder("|");- for (String key: extraIssueInfo.keySet())- builder.append(key).append("|");- builder.append("\n|");- extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));- builder.append("\n|");- for (String value: extraIssueInfo.values())- builder.append(value).append("|");-- if (issue.getDescription() != null)- issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());- else- issue.setDescription(builder.toString());- }- issues.add(issue);- }- logger.log("Imported " + numOfImportedIssues.addAndGet(pageData.size()) + " issues");- }-- };-- apiEndpoint = server.getApiEndpoint("/admin/projects/" + youTrackProjectId + "/issues?fields=" + fields);- list(client, apiEndpoint, pageDataConsumer, logger);-- if (!dryRun) {- ReferenceMigrator migrator = new ReferenceMigrator(Issue.class, issueNumberMappings);- Dao dao = OneDev.getInstance(Dao.class);- for (Issue issue: issues) {- if (issue.getDescription() != null)- issue.setDescription(migrator.migratePrefixed(issue.getDescription(), youTrackProjectShortName + "-"));-- getIssueManager().save(issue);- for (IssueSchedule schedule: issue.getSchedules())- dao.persist(schedule);- for (IssueField field: issue.getFields())- dao.persist(field);- for (IssueComment comment: issue.getComments()) {- comment.setContent(migrator.migratePrefixed(comment.getContent(), youTrackProjectShortName + "-"));- dao.persist(comment);- }- }-- for (Map.Entry<Long, Pair<LinkSpec, List<Long>>> entry: issueLinkInfo.entrySet()) {- Issue source = issueMappings.get(entry.getKey());- if (source != null) {- for (Long targetNumber: entry.getValue().getSecond()) {- Issue target = issueMappings.get(targetNumber);- if (target != null) {- IssueLink link = new IssueLink();- link.setSource(source);- link.setTarget(target);- link.setSpec(entry.getValue().getFirst());- OneDev.getInstance(IssueLinkManager.class).save(link);- }- }- }- }- }-- ImportResult result = new ImportResult();- result.mismatchedIssueFields.putAll(mismatchedIssueFields);- result.nonExistentLogins.addAll(nonExistentLogins);- result.tooLargeAttachments.addAll(tooLargeAttachments);- result.unmappedIssueFields.addAll(unmappedIssueFields);- result.unmappedIssueStates.addAll(unmappedIssueStates);- result.unmappedIssueLinks.addAll(unmappedIssueLinks);- result.unmappedIssueTags.addAll(unmappedIssueTags);-- return result;- } finally {- client.close();- }- }-- private static IssueManager getIssueManager() {- return OneDev.getInstance(IssueManager.class);- }-- static GlobalIssueSetting getIssueSetting() {- return OneDev.getInstance(SettingManager.class).getIssueSetting();- }-- static List<JsonNode> list(Client client, String apiEndpoint, TaskLogger logger) {- List<JsonNode> result = new ArrayList<>();- list(client, apiEndpoint, new PageDataConsumer() {-- @Override- public void consume(List<JsonNode> pageData) {- result.addAll(pageData);- }-- }, logger);- return result;- }-- static void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer, TaskLogger logger) {- URI uri;- try {- uri = new URIBuilder(apiEndpoint)- .addParameter("$top", String.valueOf(PER_PAGE)).build();- } catch (URISyntaxException e) {- throw new RuntimeException(e);- }-- int page = 0;- while (true) {- try {- URIBuilder builder = new URIBuilder(uri);- builder.addParameter("$skip", String.valueOf(page*PER_PAGE));- List<JsonNode> pageData = new ArrayList<>();- for (JsonNode each: JerseyUtils.get(client, builder.build().toString(), logger))- pageData.add(each);- pageDataConsumer.consume(pageData);- if (pageData.size() < PER_PAGE)- break;- page++;- } catch (URISyntaxException|InterruptedException e) {- throw new RuntimeException(e);- }- }- }--}\ No newline at end of filediff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ProjectMapping.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ProjectMapping.javaindex b693e17195..c77c00b07e 100644--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ProjectMapping.java+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ProjectMapping.java@@ -5,10 +5,11 @@ import java.util.List;import org.hibernate.validator.constraints.NotEmpty;+import io.onedev.server.util.ComponentContext;import io.onedev.server.util.validation.annotation.ProjectPath;+import io.onedev.server.web.editable.BeanEditor;import io.onedev.server.web.editable.annotation.ChoiceProvider;import io.onedev.server.web.editable.annotation.Editable;-import io.onedev.server.web.util.WicketUtils;@Editablepublic class ProjectMapping implements Serializable {@@ -47,7 +48,9 @@ public class ProjectMapping implements Serializable {@SuppressWarnings("unused")private static List<String> getProjectChoices() {- return WicketUtils.getPage().getMetaData(ImportServer.META_DATA_KEY).getProjectChoices();+ BeanEditor editor = ComponentContext.get().getComponent().findParent(BeanEditor.class);+ ImportProjects setting = (ImportProjects) editor.getModelObject();+ return setting.server.listProjects();}}\ No newline at end of filediff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImporter.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImporter.javaindex dad379ccf7..8818266be9 100644--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImporter.java+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImporter.java@@ -1,73 +1,88 @@package io.onedev.server.plugin.imports.youtrack;-import static io.onedev.server.plugin.imports.youtrack.ImportUtils.NAME;-import static io.onedev.server.plugin.imports.youtrack.ImportUtils.importIssues;-import static io.onedev.server.plugin.imports.youtrack.ImportUtils.list;--import java.util.ArrayList;+import java.io.Serializable;import java.util.List;-import javax.ws.rs.client.Client;--import com.fasterxml.jackson.databind.JsonNode;+import com.google.common.collect.Lists;-import io.onedev.commons.utils.ExplicitException;import io.onedev.commons.utils.TaskLogger;import io.onedev.server.imports.IssueImporter;import io.onedev.server.model.Project;-import io.onedev.server.web.util.WicketUtils;+import io.onedev.server.web.util.ImportStep;-public class YouTrackIssueImporter extends IssueImporter<ImportServer, IssueImportSource, ImportOption> {+public class YouTrackIssueImporter implements IssueImporter {private static final long serialVersionUID = 1L;+ private final ImportStep<ImportServer> serverStep = new ImportStep<ImportServer>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Authenticate to YouTrack";+ }++ @Override+ protected ImportServer newSetting() {+ return new ImportServer();+ }++ };++ private final ImportStep<ImportProject> projectStep = new ImportStep<ImportProject>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Choose project";+ }++ @Override+ protected ImportProject newSetting() {+ ImportProject project = new ImportProject();+ project.server = serverStep.getSetting();+ return project;+ }++ };++ private final ImportStep<ImportOption> optionStep = new ImportStep<ImportOption>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Specify import option";+ }++ @Override+ protected ImportOption newSetting() {+ ImportServer server = serverStep.getSetting();+ String project = projectStep.getSetting().getProject();+ return server.buildImportOption(project);+ }++ };+@Overridepublic String getName() {- return NAME;+ return YouTrackPluginModule.NAME;}@Override- public String doImport(ImportServer where, IssueImportSource what, ImportOption how, Project project,- boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {- logger.log("Importing issues from '" + what.getProject() + "'...");- Client client = where.newClient();- try {- String apiEndpoint = where.getApiEndpoint("/admin/projects?fields=id,name");- for (JsonNode projectNode: list(client, apiEndpoint, logger)) {- if (what.getProject().equals(projectNode.get("name").asText())) {- ImportResult result = importIssues(where, projectNode.get("id").asText(),- project, retainIssueNumbers, how, dryRun, logger);- return result.toHtml("Issues imported successfully");- }- }- throw new ExplicitException("Unable to find YouTrack project: " + what.getProject());- } finally {- client.close();- }+ public String doImport(Project project, boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {+ ImportServer server = serverStep.getSetting();+ String youTrackProject = projectStep.getSetting().getProject();+ ImportOption option = optionStep.getSetting();++ return server.importIssues(project, youTrackProject, option, retainIssueNumbers, dryRun, logger);}@Override- public IssueImportSource getWhat(ImportServer where, TaskLogger logger) {- WicketUtils.getPage().setMetaData(ImportServer.META_DATA_KEY, where);- return new IssueImportSource();+ public List<ImportStep<? extends Serializable>> getSteps() {+ return Lists.newArrayList(serverStep, projectStep, optionStep);}- @Override- public ImportOption getHow(ImportServer where, IssueImportSource what, TaskLogger logger) {- Client client = where.newClient();- try {- String apiEndpoint = where.getApiEndpoint("/admin/projects?fields=id,name,customFields(field(name),bundle(values(name)))");- for (JsonNode projectNode: list(client, apiEndpoint, logger)) {- if (what.getProject().equals(projectNode.get("name").asText())) {- List<JsonNode> projectNodes = new ArrayList<>();- projectNodes.add(projectNode);- return ImportUtils.buildImportOption(where, projectNodes, logger);- }- }- throw new ExplicitException("Unable to find YouTrack project: " + what.getProject());- } finally {- client.close();- }- }-}\ No newline at end of filediff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackPluginModule.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackPluginModule.javaindex c7482489dc..b9e4de1089 100644--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackPluginModule.java+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackPluginModule.java@@ -1,6 +1,5 @@package io.onedev.server.plugin.imports.youtrack;-import java.io.Serializable;import java.util.Collection;import com.google.common.collect.Lists;@@ -17,6 +16,8 @@ import io.onedev.server.imports.ProjectImporterContribution;*/public class YouTrackPluginModule extends AbstractPluginModule {+ static final String NAME = "YouTrack";+@Overrideprotected void configure() {super.configure();@@ -25,7 +26,7 @@ public class YouTrackPluginModule extends AbstractPluginModule {contribute(ProjectImporterContribution.class, new ProjectImporterContribution() {@Override- public Collection<ProjectImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> getImporters() {+ public Collection<ProjectImporter> getImporters() {return Lists.newArrayList(new YouTrackProjectImporter());}@@ -39,7 +40,7 @@ public class YouTrackPluginModule extends AbstractPluginModule {contribute(IssueImporterContribution.class, new IssueImporterContribution() {@Override- public Collection<IssueImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> getImporters() {+ public Collection<IssueImporter> getImporters() {return Lists.newArrayList(new YouTrackIssueImporter());}diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImporter.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImporter.javaindex 533aa537a9..2aefd4c1a7 100644--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImporter.java+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImporter.java@@ -1,119 +1,86 @@package io.onedev.server.plugin.imports.youtrack;-import static io.onedev.server.plugin.imports.youtrack.ImportUtils.list;--import java.util.ArrayList;-import java.util.Collection;-import java.util.HashMap;+import java.io.Serializable;import java.util.List;-import java.util.Map;-import java.util.Set;-import java.util.stream.Collectors;--import javax.ws.rs.client.Client;-import com.fasterxml.jackson.databind.JsonNode;-import com.google.common.base.Preconditions;+import com.google.common.collect.Lists;-import io.onedev.commons.utils.ExceptionUtils;-import io.onedev.commons.utils.ExplicitException;import io.onedev.commons.utils.TaskLogger;-import io.onedev.server.OneDev;-import io.onedev.server.entitymanager.ProjectManager;import io.onedev.server.imports.ProjectImporter;-import io.onedev.server.model.Project;-import io.onedev.server.storage.StorageManager;-import io.onedev.server.web.util.WicketUtils;+import io.onedev.server.web.util.ImportStep;-public class YouTrackProjectImporter extends ProjectImporter<ImportServer, ProjectImportSource, ImportOption> {+public class YouTrackProjectImporter implements ProjectImporter {private static final long serialVersionUID = 1L;+ private final ImportStep<ImportServer> serverStep = new ImportStep<ImportServer>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Authenticate to YouTrack";+ }++ @Override+ protected ImportServer newSetting() {+ return new ImportServer();+ }++ };++ private final ImportStep<ImportProjects> projectsStep = new ImportStep<ImportProjects>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Specify projects";+ }++ @Override+ protected ImportProjects newSetting() {+ ImportProjects projects = new ImportProjects();+ projects.server = serverStep.getSetting();+ return projects;+ }++ };++ private final ImportStep<ImportOption> optionStep = new ImportStep<ImportOption>() {++ private static final long serialVersionUID = 1L;++ @Override+ public String getTitle() {+ return "Specify import option";+ }++ @Override+ protected ImportOption newSetting() {+ ImportServer server = serverStep.getSetting();+ ImportProjects projects = projectsStep.getSetting();+ return server.buildImportOption(projects);+ }++ };+@Overridepublic String getName() {- return ImportUtils.NAME;+ return YouTrackPluginModule.NAME;}@Override- public String doImport(ImportServer where, ProjectImportSource what, ImportOption how,- boolean dryRun, TaskLogger logger) {- Map<String, String> youTrackProjectIds = new HashMap<>();- Map<String, String> youTrackProjectDescriptions = new HashMap<>();- Collection<Long> projectIds = new ArrayList<>();-- Client client = where.newClient();- try {- String apiEndpoint = where.getApiEndpoint("/admin/projects?fields=id,name,description");- for (JsonNode projectNode: list(client, apiEndpoint, logger)) {- String projectName = projectNode.get("name").asText();- youTrackProjectIds.put(projectName, projectNode.get("id").asText());- youTrackProjectDescriptions.put(projectName, projectNode.get("description").asText(null));- }-- ImportResult result = new ImportResult();-- for (ProjectMapping projectMapping: what.getProjectMappings()) {- logger.log("Importing project '" + projectMapping.getYouTrackProject() + "'...");-- String youTrackProjectId = youTrackProjectIds.get(projectMapping.getYouTrackProject());- if (youTrackProjectId == null)- throw new ExplicitException("Unable to find YouTrack project: " + projectMapping.getYouTrackProject());-- ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);- Project project = projectManager.initialize(projectMapping.getOneDevProject());- Preconditions.checkState(project.isNew());- project.setDescription(youTrackProjectDescriptions.get(projectMapping.getYouTrackProject()));- project.setIssueManagement(true);-- if (!dryRun) {- projectManager.create(project);- projectIds.add(project.getId());- }-- ImportResult currentResult = ImportUtils.importIssues(where, youTrackProjectId, project,- true, how, dryRun, logger);- result.mismatchedIssueFields.putAll(currentResult.mismatchedIssueFields);- result.nonExistentLogins.addAll(currentResult.nonExistentLogins);- result.tooLargeAttachments.addAll(currentResult.tooLargeAttachments);- result.unmappedIssueFields.addAll(currentResult.unmappedIssueFields);- result.unmappedIssueStates.addAll(currentResult.unmappedIssueStates);- result.unmappedIssueLinks.addAll(currentResult.unmappedIssueLinks);- result.unmappedIssueTags.addAll(currentResult.unmappedIssueTags);- }-- return result.toHtml("Projects imported successfully");- } catch (Exception e) {- for (Long projectId: projectIds)- OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);- throw ExceptionUtils.unchecked(e);- } finally {- client.close();- }+ public String doImport(boolean dryRun, TaskLogger logger) {+ ImportServer server = serverStep.getSetting();+ ImportProjects projects = projectsStep.getSetting();+ ImportOption option = optionStep.getSetting();+ return server.importProjects(projects, option, dryRun, logger);}@Override- public ProjectImportSource getWhat(ImportServer where, TaskLogger logger) {- WicketUtils.getPage().setMetaData(ImportServer.META_DATA_KEY, where);- return new ProjectImportSource();+ public List<ImportStep<? extends Serializable>> getSteps() {+ return Lists.newArrayList(serverStep, projectsStep, optionStep);}- @Override- public ImportOption getHow(ImportServer where, ProjectImportSource what, TaskLogger logger) {- Client client = where.newClient();- try {- List<JsonNode> projectNodes = new ArrayList<>();- String apiEndpoint = where.getApiEndpoint("/admin/projects?fields=id,name,customFields(field(name),bundle(values(name)))");- Set<String> projectNames = what.getProjectMappings().stream()- .map(it->it.getYouTrackProject()).collect(Collectors.toSet());- for (JsonNode projectNode: list(client, apiEndpoint, logger)) {- if (projectNames.contains(projectNode.get("name").asText()))- projectNodes.add(projectNode);- }-- return ImportUtils.buildImportOption(where, projectNodes, logger);- } finally {- client.close();- }- }-}\ No newline at end of filediff --git a/server-product/pom.xml b/server-product/pom.xmlindex 6fb6f0d8c1..a5682ee894 100644--- a/server-product/pom.xml+++ b/server-product/pom.xml@@ -127,17 +127,17 @@</dependency><dependency><groupId>io.onedev</groupId>- <artifactId>server-plugin-import-youtrack</artifactId>+ <artifactId>server-plugin-import-bitbucketcloud</artifactId><version>${project.version}</version></dependency><dependency><groupId>io.onedev</groupId>- <artifactId>server-plugin-import-jiracloud</artifactId>+ <artifactId>server-plugin-import-youtrack</artifactId><version>${project.version}</version></dependency><dependency><groupId>io.onedev</groupId>- <artifactId>server-plugin-import-bitbucketcloud</artifactId>+ <artifactId>server-plugin-import-jiracloud</artifactId><version>${project.version}</version></dependency><dependency>