Skip to content

The Unified and Autonomous Development Platform

Fix issue #822 - Filter pojects/repositories by group/organization when importing from GitHub/GitLab/Gitea

Robin Shena81e1fb22413
diff --git a/server-core/src/main/java/io/onedev/server/imports/Importer.java b/server-core/src/main/java/io/onedev/server/imports/Importer.java
deleted file mode 100644
index 753a49d4a9..0000000000
--- a/server-core/src/main/java/io/onedev/server/imports/Importer.java
+++ /dev/null
@@ -1,45 +0,0 @@
-package io.onedev.server.imports;
-
-import java.io.Serializable;
-import java.util.List;
-
-import io.onedev.commons.utils.TaskLogger;
-import io.onedev.server.util.ReflectionUtils;
-
-public abstract class Importer<Where extends Serializable, What extends Serializable, How extends Serializable> implements Serializable {
-
- private static final long serialVersionUID = 1L;
-
- private final Class<Where> whereClass;
-
- private final Class<What> whatClass;
-
- private final Class<How> howClass;
-
- @SuppressWarnings("unchecked")
- public Importer() {
- List<Class<?>> typeArguments = ReflectionUtils.getTypeArguments(Importer.class, getClass());
- whereClass = (Class<Where>) typeArguments.get(0);
- whatClass = (Class<What>) typeArguments.get(1);
- howClass = (Class<How>) typeArguments.get(2);
- }
-
- public Class<Where> getWhereClass() {
- return whereClass;
- }
-
- public Class<What> getWhatClass() {
- return whatClass;
- }
-
- public Class<How> getHowClass() {
- return howClass;
- }
-
- public abstract String getName();
-
- public abstract What getWhat(Where where, TaskLogger logger);
-
- public abstract How getHow(Where where, What what, TaskLogger logger);
-
-}
diff --git a/server-core/src/main/java/io/onedev/server/imports/IssueImporter.java b/server-core/src/main/java/io/onedev/server/imports/IssueImporter.java
index aa48d238f1..4b51085646 100644
--- a/server-core/src/main/java/io/onedev/server/imports/IssueImporter.java
+++ b/server-core/src/main/java/io/onedev/server/imports/IssueImporter.java
@@ -1,19 +1,21 @@
package io.onedev.server.imports;
import java.io.Serializable;
+import java.util.List;
import javax.annotation.Nullable;
import io.onedev.commons.utils.TaskLogger;
import io.onedev.server.model.Project;
+import io.onedev.server.web.util.ImportStep;
-public abstract class IssueImporter<Where extends Serializable, What extends Serializable, How extends Serializable>
- extends Importer<Where, What, How> {
+public interface IssueImporter extends Serializable {
- private static final long serialVersionUID = 1L;
+ String getName();
+
+ List<ImportStep<? extends Serializable>> getSteps();
@Nullable
- public abstract String doImport(Where where, What what, How how, Project project,
- boolean retainIssueNumbers, boolean dryRun, TaskLogger logger);
+ public abstract String doImport(Project project, boolean retainIssueNumbers, boolean dryRun, TaskLogger logger);
}
diff --git a/server-core/src/main/java/io/onedev/server/imports/IssueImporterContribution.java b/server-core/src/main/java/io/onedev/server/imports/IssueImporterContribution.java
index adc178abcd..98128f8c43 100644
--- a/server-core/src/main/java/io/onedev/server/imports/IssueImporterContribution.java
+++ b/server-core/src/main/java/io/onedev/server/imports/IssueImporterContribution.java
@@ -1,6 +1,5 @@
package io.onedev.server.imports;
-import java.io.Serializable;
import java.util.Collection;
import io.onedev.commons.loader.ExtensionPoint;
@@ -8,7 +7,7 @@ import io.onedev.commons.loader.ExtensionPoint;
@ExtensionPoint
public interface IssueImporterContribution {
- Collection<IssueImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> getImporters();
+ Collection<IssueImporter> getImporters();
int getOrder();
diff --git a/server-core/src/main/java/io/onedev/server/imports/ProjectImporter.java b/server-core/src/main/java/io/onedev/server/imports/ProjectImporter.java
index c1b44d7a38..1972d0288e 100644
--- a/server-core/src/main/java/io/onedev/server/imports/ProjectImporter.java
+++ b/server-core/src/main/java/io/onedev/server/imports/ProjectImporter.java
@@ -1,18 +1,20 @@
package io.onedev.server.imports;
import java.io.Serializable;
+import java.util.List;
import javax.annotation.Nullable;
import io.onedev.commons.utils.TaskLogger;
+import io.onedev.server.web.util.ImportStep;
-public abstract class ProjectImporter<Where extends Serializable, What extends Serializable, How extends Serializable>
- extends Importer<Where, What, How> {
+public interface ProjectImporter extends Serializable {
- private static final long serialVersionUID = 1L;
+ String getName();
+ List<ImportStep<? extends Serializable>> getSteps();
+
@Nullable
- public abstract String doImport(Where where, What what, How how,
- boolean dryRun, TaskLogger logger);
+ String doImport(boolean dryRun, TaskLogger logger);
}
diff --git a/server-core/src/main/java/io/onedev/server/imports/ProjectImporterContribution.java b/server-core/src/main/java/io/onedev/server/imports/ProjectImporterContribution.java
index a13ba0e915..ab04937b43 100644
--- a/server-core/src/main/java/io/onedev/server/imports/ProjectImporterContribution.java
+++ b/server-core/src/main/java/io/onedev/server/imports/ProjectImporterContribution.java
@@ -1,6 +1,5 @@
package io.onedev.server.imports;
-import java.io.Serializable;
import java.util.Collection;
import io.onedev.commons.loader.ExtensionPoint;
@@ -8,7 +7,7 @@ import io.onedev.commons.loader.ExtensionPoint;
@ExtensionPoint
public interface ProjectImporterContribution {
- Collection<ProjectImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> getImporters();
+ Collection<ProjectImporter> getImporters();
int getOrder();
diff --git a/server-core/src/main/java/io/onedev/server/maintenance/DefaultDataManager.java b/server-core/src/main/java/io/onedev/server/maintenance/DefaultDataManager.java
index abe655734b..b80253e449 100644
--- a/server-core/src/main/java/io/onedev/server/maintenance/DefaultDataManager.java
+++ b/server-core/src/main/java/io/onedev/server/maintenance/DefaultDataManager.java
@@ -63,7 +63,6 @@ import io.onedev.server.persistence.annotation.Sessional;
import io.onedev.server.persistence.annotation.Transactional;
import io.onedev.server.ssh.SshKeyUtils;
import io.onedev.server.util.init.ManualConfig;
-import io.onedev.server.util.init.Skippable;
import io.onedev.server.util.schedule.SchedulableTask;
import io.onedev.server.util.schedule.TaskScheduler;
import io.onedev.server.web.util.editablebean.NewUserBean;
@@ -137,11 +136,6 @@ public class DefaultDataManager implements DataManager, Serializable {
if (userManager.get(User.ROOT_ID) == null) {
manualConfigs.add(new ManualConfig("Create Administrator Account", null, new NewUserBean()) {
- @Override
- public Skippable getSkippable() {
- return null;
- }
-
@Override
public void complete() {
NewUserBean newUserBean = (NewUserBean) getSetting();
@@ -208,11 +202,6 @@ public class DefaultDataManager implements DataManager, Serializable {
manualConfigs.add(new ManualConfig("Specify System Setting", null,
systemSetting, excludedProps) {
- @Override
- public Skippable getSkippable() {
- return null;
- }
-
@Override
public void complete() {
settingManager.saveSystemSetting((SystemSetting) getSetting());
@@ -302,11 +291,6 @@ public class DefaultDataManager implements DataManager, Serializable {
manualConfigs.add(new ManualConfig("Specify Service Desk Setting", null,
setting.getValue(), new HashSet<>(), true) {
- @Override
- public Skippable getSkippable() {
- return null;
- }
-
@Override
public void complete() {
settingManager.saveServiceDeskSetting((ServiceDeskSetting) getSetting());
@@ -329,11 +313,6 @@ public class DefaultDataManager implements DataManager, Serializable {
} else if (setting.getValue() != null && !validator.validate(setting.getValue()).isEmpty()) {
manualConfigs.add(new ManualConfig("Specify Mail Setting", null, setting.getValue()) {
- @Override
- public Skippable getSkippable() {
- return null;
- }
-
@Override
public void complete() {
settingManager.saveMailSetting((MailSetting) getSetting());
@@ -349,11 +328,6 @@ public class DefaultDataManager implements DataManager, Serializable {
Serializable backupSetting = setting.getValue();
manualConfigs.add(new ManualConfig("Specify Backup Setting", null, backupSetting) {
- @Override
- public Skippable getSkippable() {
- return null;
- }
-
@Override
public void complete() {
settingManager.saveBackupSetting((BackupSetting) getSetting());
diff --git a/server-core/src/main/java/io/onedev/server/util/CollectionUtils.java b/server-core/src/main/java/io/onedev/server/util/CollectionUtils.java
index a52facf2bf..9acca1fb2d 100644
--- a/server-core/src/main/java/io/onedev/server/util/CollectionUtils.java
+++ b/server-core/src/main/java/io/onedev/server/util/CollectionUtils.java
@@ -49,4 +49,15 @@ public class CollectionUtils extends org.apache.commons.collections.CollectionUt
return list;
}
+ public static <K, V extends Comparable<? super V>> Map<K, V> sortByValue(Map<K, V> map) {
+ List<Map.Entry<K, V>> list = new ArrayList<>(map.entrySet());
+ list.sort(Map.Entry.comparingByValue());
+
+ Map<K, V> sorted = new LinkedHashMap<>();
+ for (Map.Entry<K, V> entry : list)
+ sorted.put(entry.getKey(), entry.getValue());
+
+ return sorted;
+ }
+
}
diff --git a/server-core/src/main/java/io/onedev/server/web/component/wizard/ManualConfigStep.java b/server-core/src/main/java/io/onedev/server/util/ManualConfigStep.java
similarity index 79%
rename from server-core/src/main/java/io/onedev/server/web/component/wizard/ManualConfigStep.java
rename to server-core/src/main/java/io/onedev/server/util/ManualConfigStep.java
index c162473f88..3ac5916886 100644
--- a/server-core/src/main/java/io/onedev/server/web/component/wizard/ManualConfigStep.java
+++ b/server-core/src/main/java/io/onedev/server/util/ManualConfigStep.java
@@ -1,4 +1,4 @@
-package io.onedev.server.web.component.wizard;
+package io.onedev.server.util;
import java.io.Serializable;
@@ -8,6 +8,7 @@ import org.apache.wicket.markup.html.form.FormComponent;
import io.onedev.server.util.init.ManualConfig;
import io.onedev.server.web.behavior.ForceOrdinaryStyleBehavior;
+import io.onedev.server.web.component.wizard.WizardStep;
import io.onedev.server.web.editable.BeanContext;
import io.onedev.server.web.editable.BeanEditor;
@@ -29,22 +30,6 @@ public class ManualConfigStep implements WizardStep {
return editor;
}
- @Override
- public Skippable getSkippable() {
- if (config.getSkippable() != null) {
- return new Skippable() {
-
- @Override
- public void skip() {
- config.getSkippable().skip();
- }
-
- };
- } else {
- return null;
- }
- }
-
@Override
public void complete() {
config.complete();
@@ -60,4 +45,9 @@ public class ManualConfigStep implements WizardStep {
public String getDescription() {
return config.getDescription();
}
+
+ @Override
+ public void init() {
+ }
+
}
diff --git a/server-core/src/main/java/io/onedev/server/util/init/ManualConfig.java b/server-core/src/main/java/io/onedev/server/util/init/ManualConfig.java
index 7e609256e9..6707dda616 100644
--- a/server-core/src/main/java/io/onedev/server/util/init/ManualConfig.java
+++ b/server-core/src/main/java/io/onedev/server/util/init/ManualConfig.java
@@ -56,8 +56,6 @@ public abstract class ManualConfig implements Serializable {
public Collection<String> getExcludeProperties() {
return excludedProperties;
}
-
- public abstract Skippable getSkippable();
public abstract void complete();
diff --git a/server-core/src/main/java/io/onedev/server/util/init/Skippable.java b/server-core/src/main/java/io/onedev/server/util/init/Skippable.java
deleted file mode 100644
index 757e9a631d..0000000000
--- a/server-core/src/main/java/io/onedev/server/util/init/Skippable.java
+++ /dev/null
@@ -1,5 +0,0 @@
-package io.onedev.server.util.init;
-
-public interface Skippable {
- void skip();
-}
diff --git a/server-core/src/main/java/io/onedev/server/web/component/imports/ImportPanel.html b/server-core/src/main/java/io/onedev/server/web/component/imports/ImportPanel.html
deleted file mode 100644
index 746da03274..0000000000
--- a/server-core/src/main/java/io/onedev/server/web/component/imports/ImportPanel.html
+++ /dev/null
@@ -1,13 +0,0 @@
-<wicket:panel>
- <form wicket:id="form">
- <div wicket:id="retainNumbersNote" class="alert alert-notice alert-light mb-5">Importing issues
- into currrent project. Please note that issue numbers will only be retained if the whole
- project fork graph does not have any issues to avoid duplicate issue numbers</div>
- <div wicket:id="feedback"></div>
- <div wicket:id="editor"></div>
- <input wicket:id="back" type="button" class="btn btn-light-primary mr-1" value="Back">
- <input wicket:id="next" type="submit" class="btn btn-primary mr-1" value="Next">
- <input wicket:id="dryRun" type="submit" class="btn btn-light-primary mr-1" value="Dry Run">
- <input wicket:id="import" type="submit" class="btn btn-primary mr-1" value="Import">
- </form>
-</wicket:panel>
\ No newline at end of file
diff --git a/server-core/src/main/java/io/onedev/server/web/component/imports/ImportPanel.java b/server-core/src/main/java/io/onedev/server/web/component/imports/ImportPanel.java
deleted file mode 100644
index b4e2237d38..0000000000
--- a/server-core/src/main/java/io/onedev/server/web/component/imports/ImportPanel.java
+++ /dev/null
@@ -1,205 +0,0 @@
-package io.onedev.server.web.component.imports;
-
-import java.io.Serializable;
-import java.util.concurrent.Callable;
-
-import javax.annotation.Nullable;
-
-import org.apache.wicket.ajax.AjaxRequestTarget;
-import org.apache.wicket.ajax.attributes.AjaxRequestAttributes;
-import org.apache.wicket.ajax.markup.html.AjaxLink;
-import org.apache.wicket.ajax.markup.html.form.AjaxButton;
-import org.apache.wicket.feedback.FencedFeedbackPanel;
-import org.apache.wicket.markup.html.WebMarkupContainer;
-import org.apache.wicket.markup.html.form.Form;
-import org.apache.wicket.markup.html.panel.Panel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import io.onedev.commons.utils.TaskLogger;
-import io.onedev.server.OneDev;
-import io.onedev.server.imports.Importer;
-import io.onedev.server.persistence.TransactionManager;
-import io.onedev.server.web.ajaxlistener.ShowGlobalAjaxIndicatorListener;
-import io.onedev.server.web.component.taskbutton.TaskButton;
-import io.onedev.server.web.editable.BeanContext;
-import io.onedev.server.web.page.project.ProjectPage;
-
-@SuppressWarnings("serial")
-public abstract class ImportPanel<Where extends Serializable, What extends Serializable, How extends Serializable>
- extends Panel {
-
- private static final Logger logger = LoggerFactory.getLogger(ImportPanel.class);
-
- private Where where;
-
- private What what;
-
- private How how;
-
- public ImportPanel(String id) {
- super(id);
- }
-
- @Override
- protected void onInitialize() {
- super.onInitialize();
-
- try {
- where = getImporter().getWhereClass().newInstance();
- } catch (InstantiationException | IllegalAccessException e) {
- throw new RuntimeException(e);
- }
-
- Form<?> form = new Form<Void>("form");
- form.add(new WebMarkupContainer("retainNumbersNote")
- .setVisible(getPage() instanceof ProjectPage));
- form.add(new FencedFeedbackPanel("feedback", form));
- form.add(BeanContext.edit("editor", where));
-
- form.add(new AjaxLink<Void>("back") {
-
- @Override
- public void onClick(AjaxRequestTarget target) {
- if (how != null) {
- how = null;
- form.replace(BeanContext.edit("editor", what));
- form.get("import").setVisible(false);
- form.get("dryRun").setVisible(false);
- } else {
- what = null;
- form.replace(BeanContext.edit("editor", where));
- setVisible(false);
- }
-
- form.get("next").setVisible(true);
-
- target.add(form);
- }
-
- }.setVisible(false));
-
- form.add(new AjaxButton("next") {
-
- @Override
- protected void updateAjaxAttributes(AjaxRequestAttributes attributes) {
- super.updateAjaxAttributes(attributes);
- attributes.getAjaxCallListeners().add(new ShowGlobalAjaxIndicatorListener());
- }
-
- @Override
- protected void onSubmit(AjaxRequestTarget target, Form<?> form) {
- super.onSubmit(target, form);
-
- if (what == null) {
- what = getImporter().getWhat(where, new TaskLogger() {
-
- @Override
- public void log(String message, String sessionId) {
- logger.info(message);
- }
-
- });
- form.replace(BeanContext.edit("editor", what));
- form.get("back").setVisible(true);
- } else {
- how = getImporter().getHow(where, what, new TaskLogger() {
-
- @Override
- public void log(String message, String sessionId) {
- logger.info(message);
- }
-
- });
- form.replace(BeanContext.edit("editor", how));
- form.get("next").setVisible(false);
- form.get("import").setVisible(true);
- form.get("dryRun").setVisible(true);
- }
-
- target.add(form);
- }
-
- @Override
- protected void onError(AjaxRequestTarget target, Form<?> form) {
- super.onError(target, form);
- target.add(form);
- }
-
- });
-
- form.add(new TaskButton("import") {
-
- @Override
- protected void onCompleted(AjaxRequestTarget target, boolean successful) {
- super.onCompleted(target, successful);
-
- if (successful)
- onImportSuccessful(target);
- }
-
- @Override
- protected String runTask(TaskLogger logger) {
- return OneDev.getInstance(TransactionManager.class).call(new Callable<String>() {
-
- @Override
- public String call() throws Exception {
- return doImport(where, what, how, false, logger);
- }
-
- });
- }
-
- @Override
- protected String getTitle() {
- return "Importing from " + getImporter().getName();
- }
-
- @Override
- protected void onError(AjaxRequestTarget target, Form<?> form) {
- super.onError(target, form);
- target.add(form);
- }
-
- }.setVisible(false));
-
- form.add(new TaskButton("dryRun") {
-
- @Override
- protected String runTask(TaskLogger logger) {
- return OneDev.getInstance(TransactionManager.class).call(new Callable<String>() {
-
- @Override
- public String call() throws Exception {
- return doImport(where, what, how, true, logger);
- }
-
- });
- }
-
- @Override
- protected String getTitle() {
- return "Test importing from " + getImporter().getName();
- }
-
- @Override
- protected void onError(AjaxRequestTarget target, Form<?> form) {
- super.onError(target, form);
- target.add(form);
- }
-
- }.setVisible(false));
-
- form.setOutputMarkupId(true);
- add(form);
- }
-
- protected abstract Importer<Where, What, How> getImporter();
-
- @Nullable
- protected abstract String doImport(Where where, What what, How how,
- boolean dryRun, TaskLogger logger);
-
- protected abstract void onImportSuccessful(AjaxRequestTarget target);
-
-}
diff --git a/server-core/src/main/java/io/onedev/server/web/component/issue/list/IssueListPanel.java b/server-core/src/main/java/io/onedev/server/web/component/issue/list/IssueListPanel.java
index cb40f604f2..c3b2cd678e 100644
--- a/server-core/src/main/java/io/onedev/server/web/component/issue/list/IssueListPanel.java
+++ b/server-core/src/main/java/io/onedev/server/web/component/issue/list/IssueListPanel.java
@@ -1,6 +1,5 @@
package io.onedev.server.web.component.issue.list;
-import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
@@ -324,7 +323,7 @@ public abstract class IssueListPanel extends Panel {
@Override
protected List<MenuItem> getMenuItems(FloatingPanel dropdown) {
- Collection<IssueImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> importers = new ArrayList<>();
+ Collection<IssueImporter> importers = new ArrayList<>();
List<IssueImporterContribution> contributions =
new ArrayList<>(OneDev.getExtensions(IssueImporterContribution.class));
@@ -341,7 +340,7 @@ public abstract class IssueListPanel extends Panel {
importers.addAll(contribution.getImporters());
List<MenuItem> menuItems = new ArrayList<>();
- for (IssueImporter<? extends Serializable, ? extends Serializable, ? extends Serializable> importer: importers) {
+ for (IssueImporter importer: importers) {
menuItems.add(new MenuItem() {
@Override
diff --git a/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.java b/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.java
index c03ddeae96..4d8b13cf14 100644
--- a/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.java
+++ b/server-core/src/main/java/io/onedev/server/web/component/project/list/ProjectListPanel.java
@@ -1,7 +1,6 @@
package io.onedev.server.web.component.project.list;
import java.io.IOException;
-import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -894,7 +893,7 @@ public class ProjectListPanel extends Panel {
@Override
protected List<MenuItem> getMenuItems(FloatingPanel dropdown) {
- Collection<ProjectImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> importers = new ArrayList<>();
+ Collection<ProjectImporter> importers = new ArrayList<>();
List<ProjectImporterContribution> contributions =
new ArrayList<>(OneDev.getExtensions(ProjectImporterContribution.class));
Collections.sort(contributions, new Comparator<ProjectImporterContribution>() {
@@ -910,7 +909,7 @@ public class ProjectListPanel extends Panel {
importers.addAll(contribution.getImporters());
List<MenuItem> menuItems = new ArrayList<>();
- for (ProjectImporter<? extends Serializable, ? extends Serializable, ? extends Serializable> importer: importers) {
+ for (ProjectImporter importer: importers) {
menuItems.add(new MenuItem() {
@Override
diff --git a/server-core/src/main/java/io/onedev/server/web/component/wizard/DefaultEndActionsPanel.html b/server-core/src/main/java/io/onedev/server/web/component/wizard/DefaultEndActionsPanel.html
new file mode 100644
index 0000000000..6552445db6
--- /dev/null
+++ b/server-core/src/main/java/io/onedev/server/web/component/wizard/DefaultEndActionsPanel.html
@@ -0,0 +1,3 @@
+<wicket:panel>
+ <button type="submit" wicket:id="finish" class="btn btn-primary finish">Finish</button>
+</wicket:panel>
diff --git a/server-core/src/main/java/io/onedev/server/web/component/wizard/DefaultEndActionsPanel.java b/server-core/src/main/java/io/onedev/server/web/component/wizard/DefaultEndActionsPanel.java
new file mode 100644
index 0000000000..cd5df3414a
--- /dev/null
+++ b/server-core/src/main/java/io/onedev/server/web/component/wizard/DefaultEndActionsPanel.java
@@ -0,0 +1,46 @@
+package io.onedev.server.web.component.wizard;
+
+import org.apache.wicket.markup.html.form.Button;
+import org.apache.wicket.markup.html.panel.Panel;
+
+import io.onedev.server.OneDev;
+import io.onedev.server.persistence.TransactionManager;
+
+@SuppressWarnings("serial")
+public abstract class DefaultEndActionsPanel extends Panel {
+
+ private final WizardPanel wizard;
+
+ public DefaultEndActionsPanel(String id, WizardPanel wizard) {
+ super(id);
+ this.wizard = wizard;
+ }
+
+ @Override
+ protected void onInitialize() {
+ super.onInitialize();
+
+ add(new Button("finish") {
+
+ @Override
+ public void onSubmit() {
+ super.onSubmit();
+
+ OneDev.getInstance(TransactionManager.class).run(new Runnable() {
+
+ @Override
+ public void run() {
+ wizard.getActiveStep().complete();
+ }
+
+ });
+ finished();
+ }
+
+ });
+
+ }
+
+ protected abstract void finished();
+
+}
diff --git a/server-core/src/main/java/io/onedev/server/web/component/wizard/Skippable.java b/server-core/src/main/java/io/onedev/server/web/component/wizard/Skippable.java
deleted file mode 100644
index 6ebdd1e96c..0000000000
--- a/server-core/src/main/java/io/onedev/server/web/component/wizard/Skippable.java
+++ /dev/null
@@ -1,5 +0,0 @@
-package io.onedev.server.web.component.wizard;
-
-public interface Skippable {
- void skip();
-}
\ No newline at end of file
diff --git a/server-core/src/main/java/io/onedev/server/web/component/wizard/Wizard.html b/server-core/src/main/java/io/onedev/server/web/component/wizard/WizardPanel.html
similarity index 74%
rename from server-core/src/main/java/io/onedev/server/web/component/wizard/Wizard.html
rename to server-core/src/main/java/io/onedev/server/web/component/wizard/WizardPanel.html
index 8256951924..df61bd27f6 100644
--- a/server-core/src/main/java/io/onedev/server/web/component/wizard/Wizard.html
+++ b/server-core/src/main/java/io/onedev/server/web/component/wizard/WizardPanel.html
@@ -10,9 +10,8 @@
</div>
<div class="foot">
<button wicket:id="previous" type="button" class="btn btn-light-primary previous">Previous</button>
- <button wicket:id="skip" type="button" class="btn btn-light-primary skip">Skip</button>
<button type="submit" wicket:id="next" class="btn btn-primary next">Next</button>
- <button type="submit" wicket:id="finish" class="btn btn-primary finish">Finish</button>
+ <span wicket:id="endActions"></span>
</div>
</form>
</wicket:panel>
diff --git a/server-core/src/main/java/io/onedev/server/web/component/wizard/Wizard.java b/server-core/src/main/java/io/onedev/server/web/component/wizard/WizardPanel.java
similarity index 75%
rename from server-core/src/main/java/io/onedev/server/web/component/wizard/Wizard.java
rename to server-core/src/main/java/io/onedev/server/web/component/wizard/WizardPanel.java
index dcb96cc070..7b896873e2 100644
--- a/server-core/src/main/java/io/onedev/server/web/component/wizard/Wizard.java
+++ b/server-core/src/main/java/io/onedev/server/web/component/wizard/WizardPanel.java
@@ -10,10 +10,13 @@ package io.onedev.server.web.component.wizard;
import java.util.List;
+import org.apache.wicket.Component;
+import org.apache.wicket.behavior.Behavior;
import org.apache.wicket.feedback.FencedFeedbackPanel;
import org.apache.wicket.markup.ComponentTag;
import org.apache.wicket.markup.head.CssHeaderItem;
import org.apache.wicket.markup.head.IHeaderResponse;
+import org.apache.wicket.markup.html.WebMarkupContainer;
import org.apache.wicket.markup.html.basic.Label;
import org.apache.wicket.markup.html.form.Button;
import org.apache.wicket.markup.html.form.Form;
@@ -27,13 +30,13 @@ import io.onedev.server.OneDev;
import io.onedev.server.persistence.TransactionManager;
@SuppressWarnings("serial")
-public abstract class Wizard extends Panel {
+public abstract class WizardPanel extends Panel {
private List<? extends WizardStep> steps;
private int activeStepIndex;
- public Wizard(String id, List<? extends WizardStep> steps) {
+ public WizardPanel(String id, List<? extends WizardStep> steps) {
super(id);
Preconditions.checkArgument(steps != null && !steps.isEmpty());
@@ -72,6 +75,7 @@ public abstract class Wizard extends Panel {
});
form.add(new FencedFeedbackPanel("feedback", form));
+ getActiveStep().init();
form.add(getActiveStep().render("content"));
form.add(new Link<Void>("previous") {
@@ -95,24 +99,6 @@ public abstract class Wizard extends Panel {
tag.append("class", "disabled", " ");
}
- });
- form.add(new Link<Void>("skip") {
-
- @Override
- public void onClick() {
- getActiveStep().getSkippable().skip();
- if (activeStepIndex == steps.size() - 1)
- finished();
- else
- form.replace(getActiveStep().render("content"));
- }
-
- @Override
- protected void onConfigure() {
- super.onConfigure();
- setVisible(getActiveStep().getSkippable() != null);
- }
-
});
form.add(new Button("next") {
@@ -128,6 +114,7 @@ public abstract class Wizard extends Panel {
});
activeStepIndex++;
+ getActiveStep().init();
form.replace(getActiveStep().render("content"));
}
@@ -138,42 +125,29 @@ public abstract class Wizard extends Panel {
}
});
- form.add(new Button("finish") {
+
+ form.add(newEndActions("endActions").add(new Behavior() {
@Override
- public void onSubmit() {
- super.onSubmit();
- OneDev.getInstance(TransactionManager.class).run(new Runnable() {
-
- @Override
- public void run() {
- getActiveStep().complete();
- }
-
- });
- finished();
+ public void onConfigure(Component component) {
+ super.onConfigure(component);
+ component.setVisible(activeStepIndex == steps.size()-1);
}
- @Override
- protected void onConfigure() {
- super.onConfigure();
- setVisible(activeStepIndex == steps.size()-1);
- }
-
- });
+ }));
add(form);
}
+ protected abstract WebMarkupContainer newEndActions(String componentId);
+
@Override
public void renderHead(IHeaderResponse response) {
super.renderHead(response);
response.render(CssHeaderItem.forReference(new WizardResourceReference()));
}
- private WizardStep getActiveStep() {
+ public WizardStep getActiveStep() {
return steps.get(activeStepIndex);
}
- protected abstract void finished();
-
}
diff --git a/server-core/src/main/java/io/onedev/server/web/component/wizard/WizardStep.java b/server-core/src/main/java/io/onedev/server/web/component/wizard/WizardStep.java
index 3fdf07bb60..91dd046d53 100644
--- a/server-core/src/main/java/io/onedev/server/web/component/wizard/WizardStep.java
+++ b/server-core/src/main/java/io/onedev/server/web/component/wizard/WizardStep.java
@@ -22,7 +22,7 @@ public interface WizardStep extends Serializable {
@Nullable
String getDescription();
- Skippable getSkippable();
+ void init();
void complete();
diff --git a/server-core/src/main/java/io/onedev/server/web/component/wizard/wizard.css b/server-core/src/main/java/io/onedev/server/web/component/wizard/wizard.css
index cc626c7ccf..84f58a913d 100644
--- a/server-core/src/main/java/io/onedev/server/web/component/wizard/wizard.css
+++ b/server-core/src/main/java/io/onedev/server/web/component/wizard/wizard.css
@@ -7,7 +7,7 @@
.wizard>.body {
margin-bottom: 1.5rem;
}
-.wizard>.foot>* {
+.wizard>.foot button {
margin-right: 1rem;
}
.wizard>.foot>:last-child {
diff --git a/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImportPage.html b/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImportPage.html
index 315da2b63f..2c5e651c56 100644
--- a/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImportPage.html
+++ b/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImportPage.html
@@ -1,5 +1,11 @@
<wicket:extend>
<div class="project-import card m-2 m-sm-5">
- <div wicket:id="importer" class="card-body"></div>
+ <div class="card-body">
+ <div wicket:id="wizard"></div>
+ </div>
</div>
+ <wicket:fragment wicket:id="endActionsFrag">
+ <input wicket:id="dryRun" type="submit" class="btn btn-light-primary mr-1" value="Dry Run">
+ <input wicket:id="import" type="submit" class="btn btn-primary mr-1" value="Import">
+ </wicket:fragment>
</wicket:extend>
diff --git a/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImportPage.java b/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImportPage.java
index ae2f5e24b9..468319638a 100644
--- a/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImportPage.java
+++ b/server-core/src/main/java/io/onedev/server/web/page/project/imports/ProjectImportPage.java
@@ -1,45 +1,48 @@
package io.onedev.server.web.page.project.imports;
-import java.io.Serializable;
+import java.util.concurrent.Callable;
import org.apache.wicket.Component;
import org.apache.wicket.RestartResponseException;
import org.apache.wicket.ajax.AjaxRequestTarget;
+import org.apache.wicket.markup.html.WebMarkupContainer;
import org.apache.wicket.markup.html.basic.Label;
+import org.apache.wicket.markup.html.form.Form;
+import org.apache.wicket.markup.html.panel.Fragment;
import org.apache.wicket.request.mapper.parameter.PageParameters;
import com.google.common.collect.Lists;
import io.onedev.commons.utils.TaskLogger;
import io.onedev.server.OneDev;
-import io.onedev.server.imports.Importer;
import io.onedev.server.imports.ProjectImporter;
import io.onedev.server.imports.ProjectImporterContribution;
import io.onedev.server.model.Project;
+import io.onedev.server.persistence.TransactionManager;
import io.onedev.server.search.entity.EntitySort;
import io.onedev.server.search.entity.EntitySort.Direction;
import io.onedev.server.search.entity.project.ProjectQuery;
import io.onedev.server.security.SecurityUtils;
-import io.onedev.server.web.component.imports.ImportPanel;
+import io.onedev.server.web.component.taskbutton.TaskButton;
+import io.onedev.server.web.component.wizard.WizardPanel;
import io.onedev.server.web.page.layout.LayoutPage;
import io.onedev.server.web.page.project.ProjectListPage;
@SuppressWarnings("serial")
-public class ProjectImportPage<Where extends Serializable, What extends Serializable, How extends Serializable> extends LayoutPage {
+public class ProjectImportPage extends LayoutPage {
private static final String PARAM_IMPORTER = "importer";
- private ProjectImporter<Where, What, How> importer;
+ private ProjectImporter importer;
- @SuppressWarnings("unchecked")
public ProjectImportPage(PageParameters params) {
super(params);
String importerName = params.get(PARAM_IMPORTER).toString();
for (ProjectImporterContribution contribution: OneDev.getExtensions(ProjectImporterContribution.class)) {
- for (ProjectImporter<? extends Serializable, ? extends Serializable, ? extends Serializable> importer: contribution.getImporters()) {
+ for (ProjectImporter importer: contribution.getImporters()) {
if (importer.getName().equals(importerName)) {
- this.importer = (ProjectImporter<Where, What, How>) importer;
+ this.importer = importer;
break;
}
}
@@ -53,27 +56,81 @@ public class ProjectImportPage<Where extends Serializable, What extends Serializ
protected void onInitialize() {
super.onInitialize();
- add(new ImportPanel<Where, What, How>("importer") {
+ add(new WizardPanel("wizard", importer.getSteps()) {
@Override
- protected Importer<Where, What, How> getImporter() {
- return importer;
- }
+ protected WebMarkupContainer newEndActions(String componentId) {
+ Fragment fragment = new Fragment(componentId, "endActionsFrag", ProjectImportPage.this);
+
+ fragment.add(new TaskButton("import") {
- @Override
- protected String doImport(Where where, What what, How how,
- boolean dryRun, TaskLogger logger) {
- return importer.doImport(where, what, how, dryRun, logger);
- }
+ @Override
+ protected void onCompleted(AjaxRequestTarget target, boolean successful) {
+ super.onCompleted(target, successful);
- @Override
- protected void onImportSuccessful(AjaxRequestTarget target) {
- EntitySort sort = new EntitySort();
- sort.setField(Project.NAME_UPDATE_DATE);
- sort.setDirection(Direction.DESCENDING);
- ProjectQuery query = new ProjectQuery(null, Lists.newArrayList(sort));
- PageParameters params = ProjectListPage.paramsOf(query.toString(), 0, 0);
- throw new RestartResponseException(ProjectListPage.class, params);
+ if (successful) {
+ EntitySort sort = new EntitySort();
+ sort.setField(Project.NAME_UPDATE_DATE);
+ sort.setDirection(Direction.DESCENDING);
+ ProjectQuery query = new ProjectQuery(null, Lists.newArrayList(sort));
+ PageParameters params = ProjectListPage.paramsOf(query.toString(), 0, 0);
+ throw new RestartResponseException(ProjectListPage.class, params);
+ }
+ }
+
+ @Override
+ protected String runTask(TaskLogger logger) {
+ return OneDev.getInstance(TransactionManager.class).call(new Callable<String>() {
+
+ @Override
+ public String call() throws Exception {
+ return importer.doImport(false, logger);
+ }
+
+ });
+ }
+
+ @Override
+ protected String getTitle() {
+ return "Importing from " + importer.getName();
+ }
+
+ @Override
+ protected void onError(AjaxRequestTarget target, Form<?> form) {
+ super.onError(target, form);
+ target.add(form);
+ }
+
+ });
+
+ fragment.add(new TaskButton("dryRun") {
+
+ @Override
+ protected String runTask(TaskLogger logger) {
+ return OneDev.getInstance(TransactionManager.class).call(new Callable<String>() {
+
+ @Override
+ public String call() throws Exception {
+ return importer.doImport(true, logger);
+ }
+
+ });
+ }
+
+ @Override
+ protected String getTitle() {
+ return "Test importing from " + importer.getName();
+ }
+
+ @Override
+ protected void onError(AjaxRequestTarget target, Form<?> form) {
+ super.onError(target, form);
+ target.add(form);
+ }
+
+ });
+
+ return fragment;
}
});
diff --git a/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.html b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.html
index 01bcc25067..89e6b80468 100644
--- a/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.html
+++ b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.html
@@ -1,5 +1,14 @@
<wicket:extend>
<div class="issue-import card m-2 m-sm-5">
- <div wicket:id="importer" class="card-body"></div>
+ <div class="card-body">
+ <div class="alert alert-notice alert-light mb-5">Importing issues
+ into currrent project. Please note that issue numbers will only be retained if the whole
+ project fork graph does not have any issues to avoid duplicate issue numbers</div>
+ <div wicket:id="wizard"></div>
+ </div>
</div>
+ <wicket:fragment wicket:id="endActionsFrag">
+ <input wicket:id="dryRun" type="submit" class="btn btn-light-primary mr-1" value="Dry Run">
+ <input wicket:id="import" type="submit" class="btn btn-primary mr-1" value="Import">
+ </wicket:fragment>
</wicket:extend>
diff --git a/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.java b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.java
index 79d1facc84..ec4eed83ee 100644
--- a/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.java
+++ b/server-core/src/main/java/io/onedev/server/web/page/project/issues/imports/IssueImportPage.java
@@ -1,11 +1,15 @@
package io.onedev.server.web.page.project.issues.imports;
import java.io.Serializable;
+import java.util.concurrent.Callable;
import org.apache.wicket.Component;
import org.apache.wicket.RestartResponseException;
import org.apache.wicket.ajax.AjaxRequestTarget;
+import org.apache.wicket.markup.html.WebMarkupContainer;
import org.apache.wicket.markup.html.basic.Label;
+import org.apache.wicket.markup.html.form.Form;
+import org.apache.wicket.markup.html.panel.Fragment;
import org.apache.wicket.request.mapper.parameter.PageParameters;
import com.google.common.collect.Lists;
@@ -13,17 +17,17 @@ import com.google.common.collect.Lists;
import io.onedev.commons.utils.TaskLogger;
import io.onedev.server.OneDev;
import io.onedev.server.entitymanager.IssueManager;
-import io.onedev.server.entitymanager.ProjectManager;
-import io.onedev.server.imports.Importer;
import io.onedev.server.imports.IssueImporter;
import io.onedev.server.imports.IssueImporterContribution;
import io.onedev.server.model.Issue;
import io.onedev.server.model.Project;
+import io.onedev.server.persistence.TransactionManager;
import io.onedev.server.search.entity.EntitySort;
import io.onedev.server.search.entity.EntitySort.Direction;
import io.onedev.server.search.entity.issue.IssueQuery;
import io.onedev.server.security.SecurityUtils;
-import io.onedev.server.web.component.imports.ImportPanel;
+import io.onedev.server.web.component.taskbutton.TaskButton;
+import io.onedev.server.web.component.wizard.WizardPanel;
import io.onedev.server.web.page.project.ProjectPage;
import io.onedev.server.web.page.project.dashboard.ProjectDashboardPage;
import io.onedev.server.web.page.project.issues.list.ProjectIssueListPage;
@@ -33,17 +37,16 @@ public class IssueImportPage<Where extends Serializable, What extends Serializab
private static final String PARAM_IMPORTER = "importer";
- private IssueImporter<Where, What, How> importer;
+ private IssueImporter importer;
- @SuppressWarnings("unchecked")
public IssueImportPage(PageParameters params) {
super(params);
String importerName = params.get(PARAM_IMPORTER).toString();
for (IssueImporterContribution contribution: OneDev.getExtensions(IssueImporterContribution.class)) {
- for (IssueImporter<? extends Serializable, ? extends Serializable, ? extends Serializable> importer: contribution.getImporters()) {
+ for (IssueImporter importer: contribution.getImporters()) {
if (importer.getName().equals(importerName)) {
- this.importer = (IssueImporter<Where, What, How>) importer;
+ this.importer = importer;
break;
}
}
@@ -57,43 +60,100 @@ public class IssueImportPage<Where extends Serializable, What extends Serializab
protected void onInitialize() {
super.onInitialize();
- Long projectId = getProject().getId();
- add(new ImportPanel<Where, What, How>("importer") {
+ add(new WizardPanel("wizard", importer.getSteps()) {
@Override
- protected Importer<Where, What, How> getImporter() {
- return importer;
- }
+ protected WebMarkupContainer newEndActions(String componentId) {
+ Fragment fragment = new Fragment(componentId, "endActionsFrag", IssueImportPage.this);
+
+ fragment.add(new TaskButton("import") {
- @Override
- protected String doImport(Where where, What what, How how, boolean dryRun, TaskLogger logger) {
- IssueManager issueManager = OneDev.getInstance(IssueManager.class);
- Project project = OneDev.getInstance(ProjectManager.class).load(projectId);
- Project numberScope = project.getForkRoot();
- try {
- issueManager.resetNextNumber(numberScope);
- Long nextNumber = issueManager.getNextNumber(numberScope);
- issueManager.resetNextNumber(numberScope);
- return importer.doImport(where, what, how, project, nextNumber==1L, dryRun, logger);
- } finally {
- issueManager.resetNextNumber(numberScope);
- }
- }
+ @Override
+ protected void onCompleted(AjaxRequestTarget target, boolean successful) {
+ super.onCompleted(target, successful);
- @Override
- protected void onImportSuccessful(AjaxRequestTarget target) {
- EntitySort sort = new EntitySort();
- sort.setField(Issue.NAME_NUMBER);
- sort.setDirection(Direction.DESCENDING);
- IssueQuery query = new IssueQuery(null, Lists.newArrayList(sort));
+ if (successful) {
+ EntitySort sort = new EntitySort();
+ sort.setField(Issue.NAME_NUMBER);
+ sort.setDirection(Direction.DESCENDING);
+ IssueQuery query = new IssueQuery(null, Lists.newArrayList(sort));
+
+ PageParameters params = ProjectIssueListPage.paramsOf(getProject(), query.toString(), 0);
+ throw new RestartResponseException(ProjectIssueListPage.class, params);
+ }
+ }
+
+ @Override
+ protected String runTask(TaskLogger logger) {
+ return OneDev.getInstance(TransactionManager.class).call(new Callable<String>() {
+
+ @Override
+ public String call() throws Exception {
+ return doImport(false, logger);
+ }
+
+ });
+ }
+
+ @Override
+ protected String getTitle() {
+ return "Importing from " + importer.getName();
+ }
+
+ @Override
+ protected void onError(AjaxRequestTarget target, Form<?> form) {
+ super.onError(target, form);
+ target.add(form);
+ }
+
+ });
- PageParameters params = ProjectIssueListPage.paramsOf(getProject(), query.toString(), 0);
- throw new RestartResponseException(ProjectIssueListPage.class, params);
+ fragment.add(new TaskButton("dryRun") {
+
+ @Override
+ protected String runTask(TaskLogger logger) {
+ return OneDev.getInstance(TransactionManager.class).call(new Callable<String>() {
+
+ @Override
+ public String call() throws Exception {
+ return doImport(true, logger);
+ }
+
+ });
+ }
+
+ @Override
+ protected String getTitle() {
+ return "Test importing from " + importer.getName();
+ }
+
+ @Override
+ protected void onError(AjaxRequestTarget target, Form<?> form) {
+ super.onError(target, form);
+ target.add(form);
+ }
+
+ });
+
+ return fragment;
}
});
}
+
+ private String doImport(boolean dryRun, TaskLogger logger) {
+ IssueManager issueManager = OneDev.getInstance(IssueManager.class);
+ Project numberScope = getProject().getForkRoot();
+ try {
+ issueManager.resetNextNumber(numberScope);
+ Long nextNumber = issueManager.getNextNumber(numberScope);
+ issueManager.resetNextNumber(numberScope);
+ return importer.doImport(getProject(), nextNumber==1L, dryRun, logger);
+ } finally {
+ issueManager.resetNextNumber(numberScope);
+ }
+ }
@Override
protected boolean isPermitted() {
diff --git a/server-core/src/main/java/io/onedev/server/web/page/simple/serverinit/ServerInitPage.java b/server-core/src/main/java/io/onedev/server/web/page/simple/serverinit/ServerInitPage.java
index 19e27a0dd3..d44952cd28 100644
--- a/server-core/src/main/java/io/onedev/server/web/page/simple/serverinit/ServerInitPage.java
+++ b/server-core/src/main/java/io/onedev/server/web/page/simple/serverinit/ServerInitPage.java
@@ -14,12 +14,12 @@ import io.onedev.server.OneDev;
import io.onedev.server.entitymanager.UserManager;
import io.onedev.server.model.User;
import io.onedev.server.security.SecurityUtils;
+import io.onedev.server.util.ManualConfigStep;
import io.onedev.server.util.init.InitStage;
import io.onedev.server.util.init.ManualConfig;
-import io.onedev.server.util.init.Skippable;
import io.onedev.server.web.WebSession;
-import io.onedev.server.web.component.wizard.ManualConfigStep;
-import io.onedev.server.web.component.wizard.Wizard;
+import io.onedev.server.web.component.wizard.DefaultEndActionsPanel;
+import io.onedev.server.web.component.wizard.WizardPanel;
import io.onedev.server.web.page.project.ProjectListPage;
import io.onedev.server.web.page.simple.SimplePage;
@@ -43,26 +43,6 @@ public class ServerInitPage extends SimplePage {
clonedConfigs.add(new ManualConfig(lastConfig.getTitle(), lastConfig.getDescription(),
lastConfig.getSetting(), lastConfig.getExcludeProperties(), lastConfig.isForceOrdinaryStyle()) {
- @Override
- public Skippable getSkippable() {
- final Skippable skippable = lastConfig.getSkippable();
- if (skippable != null) {
- return new Skippable() {
-
- @Override
- public void skip() {
- skippable.skip();
- InitStage initStage = OneDev.getInstance().getInitStage();
- if (initStage != null)
- initStage.finished();
- }
-
- };
- } else {
- return null;
- }
- }
-
@Override
public void complete() {
lastConfig.complete();
@@ -92,15 +72,23 @@ public class ServerInitPage extends SimplePage {
List<ManualConfigStep> configSteps = new ArrayList<ManualConfigStep>();
for (ManualConfig each: initStage.getManualConfigs())
configSteps.add(new ManualConfigStep(each));
- add(new Wizard("wizard", configSteps) {
+ add(new WizardPanel("wizard", configSteps) {
@Override
- protected void finished() {
- WebSession.get().logout();
- User root = OneDev.getInstance(UserManager.class).getRoot();
- SecurityUtils.getSubject().runAs(root.getPrincipals());
- throw new RestartResponseException(ProjectListPage.class);
+ protected WebMarkupContainer newEndActions(String componentId) {
+ return new DefaultEndActionsPanel(componentId, this) {
+
+ @Override
+ protected void finished() {
+ WebSession.get().logout();
+ User root = OneDev.getInstance(UserManager.class).getRoot();
+ SecurityUtils.getSubject().runAs(root.getPrincipals());
+ throw new RestartResponseException(ProjectListPage.class);
+ }
+
+ };
}
+
});
} else {
diff --git a/server-core/src/main/java/io/onedev/server/web/util/ImportStep.java b/server-core/src/main/java/io/onedev/server/web/util/ImportStep.java
new file mode 100644
index 0000000000..82a1f37633
--- /dev/null
+++ b/server-core/src/main/java/io/onedev/server/web/util/ImportStep.java
@@ -0,0 +1,42 @@
+package io.onedev.server.web.util;
+
+import java.io.Serializable;
+
+import org.apache.wicket.Component;
+
+import io.onedev.server.web.component.wizard.WizardStep;
+import io.onedev.server.web.editable.BeanContext;
+
+public abstract class ImportStep<T extends Serializable> implements WizardStep {
+
+ private static final long serialVersionUID = 1L;
+
+ private T setting;
+
+ @Override
+ public Component render(String componentId) {
+ return BeanContext.edit(componentId, setting);
+ }
+
+ @Override
+ public String getDescription() {
+ return null;
+ }
+
+ protected abstract T newSetting();
+
+ @Override
+ public void init() {
+ setting = newSetting();
+ }
+
+ @Override
+ public void complete() {
+
+ }
+
+ public T getSetting() {
+ return setting;
+ }
+
+}
diff --git a/server-plugin/pom.xml b/server-plugin/pom.xml
index 35312c8259..75e39b2f41 100644
--- a/server-plugin/pom.xml
+++ b/server-plugin/pom.xml
@@ -1,5 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>server-plugin</artifactId>
<packaging>pom</packaging>
@@ -20,31 +22,31 @@
<module>server-plugin-executor-kubernetes</module>
<module>server-plugin-executor-serverdocker</module>
<module>server-plugin-buildspec-maven</module>
- <module>server-plugin-buildspec-gradle</module>
- <module>server-plugin-buildspec-node</module>
- <module>server-plugin-authenticator-ldap</module>
- <module>server-plugin-sso-openid</module>
- <module>server-plugin-report-markdown</module>
- <module>server-plugin-report-jest</module>
- <module>server-plugin-report-clover</module>
- <module>server-plugin-report-checkstyle</module>
- <module>server-plugin-import-github</module>
- <module>server-plugin-import-youtrack</module>
- <module>server-plugin-import-gitlab</module>
- <module>server-plugin-import-gitea</module>
- <module>server-plugin-import-bitbucketcloud</module>
- <module>server-plugin-import-jiracloud</module>
- <module>server-plugin-executor-remotedocker</module>
- <module>server-plugin-executor-servershell</module>
- <module>server-plugin-executor-remoteshell</module>
- <module>server-plugin-report-unittest</module>
- <module>server-plugin-report-junit</module>
- <module>server-plugin-report-coverage</module>
- <module>server-plugin-report-jacoco</module>
- <module>server-plugin-report-problem</module>
- <module>server-plugin-report-spotbugs</module>
- <module>server-plugin-report-pmd</module>
- <module>server-plugin-report-cpd</module>
- <module>server-plugin-sso-discord</module>
- </modules>
+ <module>server-plugin-buildspec-gradle</module>
+ <module>server-plugin-buildspec-node</module>
+ <module>server-plugin-authenticator-ldap</module>
+ <module>server-plugin-sso-openid</module>
+ <module>server-plugin-report-markdown</module>
+ <module>server-plugin-report-jest</module>
+ <module>server-plugin-report-clover</module>
+ <module>server-plugin-report-checkstyle</module>
+ <module>server-plugin-import-github</module>
+ <module>server-plugin-import-gitlab</module>
+ <module>server-plugin-import-gitea</module>
+ <module>server-plugin-import-bitbucketcloud</module>
+ <module>server-plugin-import-jiracloud</module>
+ <module>server-plugin-import-youtrack</module>
+ <module>server-plugin-executor-remotedocker</module>
+ <module>server-plugin-executor-servershell</module>
+ <module>server-plugin-executor-remoteshell</module>
+ <module>server-plugin-report-unittest</module>
+ <module>server-plugin-report-junit</module>
+ <module>server-plugin-report-coverage</module>
+ <module>server-plugin-report-jacoco</module>
+ <module>server-plugin-report-problem</module>
+ <module>server-plugin-report-spotbugs</module>
+ <module>server-plugin-report-pmd</module>
+ <module>server-plugin-report-cpd</module>
+ <module>server-plugin-sso-discord</module>
+ </modules>
</project>
\ No newline at end of file
diff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketPluginModule.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketPluginModule.java
index 82b42ed1bb..9395354c07 100644
--- a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketPluginModule.java
+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketPluginModule.java
@@ -1,6 +1,5 @@
package io.onedev.server.plugin.imports.bitbucketcloud;
-import java.io.Serializable;
import java.util.Collection;
import com.google.common.collect.Lists;
@@ -15,6 +14,9 @@ import io.onedev.server.imports.ProjectImporterContribution;
*/
public class BitbucketPluginModule extends AbstractPluginModule {
+
+ static final String NAME = "Bitbucket Cloud";
+
@Override
protected void configure() {
super.configure();
@@ -23,7 +25,7 @@ public class BitbucketPluginModule extends AbstractPluginModule {
contribute(ProjectImporterContribution.class, new ProjectImporterContribution() {
@Override
- public Collection<ProjectImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> getImporters() {
+ public Collection<ProjectImporter> getImporters() {
return Lists.newArrayList(new BitbucketProjectImporter());
}
diff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketProjectImporter.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketProjectImporter.java
index 5542851687..753b3009e8 100644
--- a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketProjectImporter.java
+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/BitbucketProjectImporter.java
@@ -1,106 +1,108 @@
package io.onedev.server.plugin.imports.bitbucketcloud;
-import java.util.ArrayList;
-import java.util.Collection;
+import java.io.Serializable;
+import java.util.List;
-import javax.ws.rs.client.Client;
+import com.google.common.collect.Lists;
-import org.apache.http.client.utils.URIBuilder;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.google.common.base.Preconditions;
-
-import io.onedev.commons.utils.ExplicitException;
import io.onedev.commons.utils.TaskLogger;
-import io.onedev.server.OneDev;
-import io.onedev.server.entitymanager.ProjectManager;
import io.onedev.server.imports.ProjectImporter;
-import io.onedev.server.model.Project;
-import io.onedev.server.storage.StorageManager;
-import io.onedev.server.util.JerseyUtils;
+import io.onedev.server.web.util.ImportStep;
-public class BitbucketProjectImporter extends ProjectImporter<ImportServer, ProjectImportSource, ProjectImportOption> {
+public class BitbucketProjectImporter implements ProjectImporter {
private static final long serialVersionUID = 1L;
- @Override
- public String getName() {
- return ImportUtils.NAME;
- }
+ private final ImportStep<ImportServer> serverStep = new ImportStep<ImportServer>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Authenticate to Bitbucket Cloud";
+ }
+
+ @Override
+ protected ImportServer newSetting() {
+ return new ImportServer();
+ }
+
+ };
- @Override
- public String doImport(ImportServer where, ProjectImportSource what, ProjectImportOption how,
- boolean dryRun, TaskLogger logger) {
- Collection<Long> projectIds = new ArrayList<>();
- Client client = where.newClient();
- try {
- for (ProjectMapping projectMapping: what.getProjectMappings()) {
- logger.log("Cloning code from repository " + projectMapping.getBitbucketRepo() + "...");
-
- String apiEndpoint = where.getApiEndpoint("/repositories/" + projectMapping.getBitbucketRepo());
- JsonNode repoNode = JerseyUtils.get(client, apiEndpoint, logger);
- ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
- Project project = projectManager.initialize(projectMapping.getOneDevProject());
- Preconditions.checkState(project.isNew());
-
- project.setDescription(repoNode.get("description").asText(null));
-
- boolean isPrivate = repoNode.get("is_private").asBoolean();
- if (!isPrivate && how.getPublicRole() != null)
- project.setDefaultRole(how.getPublicRole());
-
- String cloneUrl = null;
- for (JsonNode cloneNode: repoNode.get("links").get("clone")) {
- if (cloneNode.get("name").asText().equals("https")) {
- cloneUrl = cloneNode.get("href").asText();
- break;
- }
- }
- if (cloneUrl == null)
- throw new ExplicitException("Https clone url not found");
-
- URIBuilder builder = new URIBuilder(cloneUrl);
- if (isPrivate)
- builder.setUserInfo(where.getUserName(), where.getAppPassword());
-
- if (!dryRun) {
- projectManager.clone(project, builder.build().toString());
- projectIds.add(project.getId());
- }
+ private final ImportStep<ImportWorkspace> workspaceStep = new ImportStep<ImportWorkspace>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Choose workspace";
+ }
+
+ @Override
+ protected ImportWorkspace newSetting() {
+ ImportWorkspace workspace = new ImportWorkspace();
+ workspace.server = serverStep.getSetting();
+ return workspace;
+ }
+
+ };
+
+ private final ImportStep<ImportRepositories> repositoriesStep = new ImportStep<ImportRepositories>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Specify repositories";
+ }
+
+ @Override
+ protected ImportRepositories newSetting() {
+ ImportRepositories repositories = new ImportRepositories();
+ String workspace = workspaceStep.getSetting().getWorkspace();
+ for (String repository: serverStep.getSetting().listRepositories(workspace)) {
+ ProjectMapping projectMapping = new ProjectMapping();
+ projectMapping.setBitbucketRepo(repository);
+ projectMapping.setOneDevProject(repository);
+ repositories.getProjectMappings().add(projectMapping);
}
- return "Repositories imported successfully";
- } catch (Exception e) {
- for (Long projectId: projectIds)
- OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);
- throw new RuntimeException(e);
- } finally {
- client.close();
+ return repositories;
}
- }
+
+ };
+
+ private final ImportStep<ImportOption> optionStep = new ImportStep<ImportOption>() {
- @Override
- public ProjectImportSource getWhat(ImportServer where, TaskLogger logger) {
- ProjectImportSource importSource = new ProjectImportSource();
- Client client = where.newClient();
- try {
- String apiEndpoint = where.getApiEndpoint("/repositories?role=member");
- for (JsonNode repoNode: ImportUtils.list(client, apiEndpoint, logger)) {
- String fullName = repoNode.get("full_name").asText();
- ProjectMapping projectMapping = new ProjectMapping();
- projectMapping.setBitbucketRepo(fullName);
- projectMapping.setOneDevProject(fullName);
- importSource.getProjectMappings().add(projectMapping);
- }
- } finally {
- client.close();
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Specify import option";
+ }
+
+ @Override
+ protected ImportOption newSetting() {
+ return new ImportOption();
}
- return importSource;
+
+ };
+
+ @Override
+ public String getName() {
+ return BitbucketPluginModule.NAME;
+ }
+
+ @Override
+ public String doImport(boolean dryRun, TaskLogger logger) {
+ ImportRepositories repositories = repositoriesStep.getSetting();
+ ImportOption option = optionStep.getSetting();
+ return serverStep.getSetting().doImport(repositories, option, dryRun, logger);
}
@Override
- public ProjectImportOption getHow(ImportServer where, ProjectImportSource what, TaskLogger logger) {
- return new ProjectImportOption();
+ public List<ImportStep<? extends Serializable>> getSteps() {
+ return Lists.newArrayList(serverStep, workspaceStep, repositoriesStep, optionStep);
}
-
+
}
\ No newline at end of file
diff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectImportOption.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportOption.java
similarity index 94%
rename from server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectImportOption.java
rename to server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportOption.java
index 44a12ce103..4cdff02305 100644
--- a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectImportOption.java
+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportOption.java
@@ -11,7 +11,7 @@ import io.onedev.server.web.editable.annotation.Editable;
import io.onedev.server.web.editable.annotation.RoleChoice;
@Editable
-public class ProjectImportOption implements Serializable {
+public class ImportOption implements Serializable {
private static final long serialVersionUID = 1L;
diff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectImportSource.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportRepositories.java
similarity index 94%
rename from server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectImportSource.java
rename to server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportRepositories.java
index f58970685e..bc8c999a9e 100644
--- a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ProjectImportSource.java
+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportRepositories.java
@@ -15,10 +15,11 @@ import io.onedev.server.model.Project;
import io.onedev.server.util.validation.Validatable;
import io.onedev.server.util.validation.annotation.ClassValidating;
import io.onedev.server.web.editable.annotation.Editable;
+import io.onedev.server.web.editable.annotation.OmitName;
@Editable
@ClassValidating
-public class ProjectImportSource implements Serializable, Validatable {
+public class ImportRepositories implements Serializable, Validatable {
private static final long serialVersionUID = 1L;
@@ -26,6 +27,7 @@ public class ProjectImportSource implements Serializable, Validatable {
@Editable(order=100, name="Repositories to Import")
@Size(min=1, max=10000, message="No repositories to import")
+ @OmitName
public List<ProjectMapping> getProjectMappings() {
return projectMappings;
}
diff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportServer.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportServer.java
index 8fd6bbf019..fe452bd0e8 100644
--- a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportServer.java
+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportServer.java
@@ -1,6 +1,13 @@
package io.onedev.server.plugin.imports.bitbucketcloud;
import java.io.Serializable;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import javax.validation.ConstraintValidatorContext;
import javax.ws.rs.client.Client;
@@ -9,13 +16,26 @@ import javax.ws.rs.client.Invocation;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response;
-import org.apache.wicket.MetaDataKey;
+import org.apache.http.client.utils.URIBuilder;
import org.glassfish.jersey.client.ClientProperties;
import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
import org.hibernate.validator.constraints.NotEmpty;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.base.Preconditions;
+
+import io.onedev.commons.utils.ExplicitException;
import io.onedev.commons.utils.StringUtils;
+import io.onedev.commons.utils.TaskLogger;
+import io.onedev.server.OneDev;
+import io.onedev.server.entitymanager.ProjectManager;
+import io.onedev.server.model.Project;
+import io.onedev.server.storage.StorageManager;
+import io.onedev.server.util.CollectionUtils;
import io.onedev.server.util.JerseyUtils;
+import io.onedev.server.util.JerseyUtils.PageDataConsumer;
import io.onedev.server.util.validation.Validatable;
import io.onedev.server.util.validation.annotation.ClassValidating;
import io.onedev.server.web.editable.annotation.Editable;
@@ -27,11 +47,9 @@ public class ImportServer implements Serializable, Validatable {
private static final long serialVersionUID = 1L;
- static final MetaDataKey<ImportServer> META_DATA_KEY = new MetaDataKey<ImportServer>() {
-
- private static final long serialVersionUID = 1L;
-
- };
+ private static final Logger logger = LoggerFactory.getLogger(ImportServer.class);
+
+ private static final int PER_PAGE = 50;
private String userName;
@@ -59,10 +77,159 @@ public class ImportServer implements Serializable, Validatable {
this.appPassword = appPassword;
}
- public String getApiEndpoint(String apiPath) {
+ private String getApiEndpoint(String apiPath) {
return "https://api.bitbucket.org/2.0/" + StringUtils.stripStart(apiPath, "/");
}
+
+ Map<String, String> listWorkspaces() {
+ Map<String, String> workspaces = new HashMap<>();
+
+ Client client = newClient();
+ try {
+ String apiEndpoint = getApiEndpoint("/user/permissions/workspaces");
+ for (JsonNode valueElementNode: list(client, apiEndpoint, new TaskLogger() {
+
+ @Override
+ public void log(String message, String sessionId) {
+ logger.info(message);
+ }
+
+ })) {
+ JsonNode workspaceNode = valueElementNode.get("workspace");
+ workspaces.put(workspaceNode.get("slug").asText(), workspaceNode.get("name").asText());
+ }
+ CollectionUtils.sortByValue(workspaces);
+ } catch (Exception e) {
+ logger.error("Error listing workspaces", e);
+ } finally {
+ client.close();
+ }
+
+ return workspaces;
+ }
+
+ List<String> listRepositories(String workspaceId) {
+ Client client = newClient();
+ try {
+ List<String> repositories = new ArrayList<>();
+
+ String apiEndpoint = getApiEndpoint("/repositories/" + workspaceId);
+ for (JsonNode repoNode: list(client, apiEndpoint, new TaskLogger() {
+
+ @Override
+ public void log(String message, String sessionId) {
+ logger.info(message);
+ }
+
+ })) {
+ repositories.add(repoNode.get("full_name").asText());
+ }
+
+ return repositories;
+ } finally {
+ client.close();
+ }
+ }
+
+ private List<JsonNode> list(Client client, String apiEndpoint, TaskLogger logger) {
+ List<JsonNode> result = new ArrayList<>();
+ list(client, apiEndpoint, new PageDataConsumer() {
+
+ @Override
+ public void consume(List<JsonNode> pageData) {
+ result.addAll(pageData);
+ }
+
+ }, logger);
+ return result;
+ }
+
+ private void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer,
+ TaskLogger logger) {
+ URI uri;
+ try {
+ uri = new URIBuilder(apiEndpoint)
+ .addParameter("pagelen", String.valueOf(PER_PAGE)).build();
+ } catch (URISyntaxException e) {
+ throw new RuntimeException(e);
+ }
+
+ while (true) {
+ try {
+ List<JsonNode> pageData = new ArrayList<>();
+ JsonNode resultNode = JerseyUtils.get(client, uri.toString(), logger);
+ if (resultNode.hasNonNull("values")) {
+ for (JsonNode each: resultNode.get("values"))
+ pageData.add(each);
+ pageDataConsumer.consume(pageData);
+ }
+ if (resultNode.hasNonNull("next"))
+ uri = new URIBuilder(resultNode.get("next").asText()).build();
+ else
+ break;
+ } catch (URISyntaxException | InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ String doImport(ImportRepositories repositories, ImportOption option, boolean dryRun, TaskLogger logger) {
+ Collection<Long> projectIds = new ArrayList<>();
+
+ Client client = newClient();
+ try {
+ for (ProjectMapping projectMapping: repositories.getProjectMappings()) {
+ logger.log("Cloning code from repository " + projectMapping.getBitbucketRepo() + "...");
+
+ String apiEndpoint = getApiEndpoint("/repositories/" + projectMapping.getBitbucketRepo());
+ JsonNode repoNode = JerseyUtils.get(client, apiEndpoint, logger);
+ ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
+ Project project = projectManager.initialize(projectMapping.getOneDevProject());
+ Preconditions.checkState(project.isNew());
+
+ project.setDescription(repoNode.get("description").asText(null));
+
+ boolean isPrivate = repoNode.get("is_private").asBoolean();
+ if (!isPrivate && option.getPublicRole() != null)
+ project.setDefaultRole(option.getPublicRole());
+ String cloneUrl = null;
+ for (JsonNode cloneNode: repoNode.get("links").get("clone")) {
+ if (cloneNode.get("name").asText().equals("https")) {
+ cloneUrl = cloneNode.get("href").asText();
+ break;
+ }
+ }
+ if (cloneUrl == null)
+ throw new ExplicitException("Https clone url not found");
+
+ URIBuilder builder = new URIBuilder(cloneUrl);
+ if (isPrivate)
+ builder.setUserInfo(getUserName(), getAppPassword());
+
+ if (!dryRun) {
+ projectManager.clone(project, builder.build().toString());
+ projectIds.add(project.getId());
+ }
+ }
+
+ return "Repositories imported successfully";
+ } catch (Exception e) {
+ for (Long projectId: projectIds)
+ OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);
+ throw new RuntimeException(e);
+ } finally {
+ client.close();
+ }
+ }
+
+ private Client newClient() {
+ Client client = ClientBuilder.newClient();
+ client.property(ClientProperties.FOLLOW_REDIRECTS, true);
+ client.register(HttpAuthenticationFeature.basic(getUserName(), getAppPassword()));
+ return client;
+ }
+
@Override
public boolean isValid(ConstraintValidatorContext context) {
Client client = ClientBuilder.newClient();
@@ -99,11 +266,4 @@ public class ImportServer implements Serializable, Validatable {
return true;
}
- Client newClient() {
- Client client = ClientBuilder.newClient();
- client.property(ClientProperties.FOLLOW_REDIRECTS, true);
- client.register(HttpAuthenticationFeature.basic(getUserName(), getAppPassword()));
- return client;
- }
-
}
diff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportUtils.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportUtils.java
deleted file mode 100644
index 0810d5cecc..0000000000
--- a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportUtils.java
+++ /dev/null
@@ -1,66 +0,0 @@
-package io.onedev.server.plugin.imports.bitbucketcloud;
-
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.List;
-
-import javax.ws.rs.client.Client;
-
-import org.apache.http.client.utils.URIBuilder;
-
-import com.fasterxml.jackson.databind.JsonNode;
-
-import io.onedev.commons.utils.TaskLogger;
-import io.onedev.server.util.JerseyUtils;
-import io.onedev.server.util.JerseyUtils.PageDataConsumer;
-
-public class ImportUtils {
-
- static final String NAME = "Bitbucket Cloud";
-
- static final int PER_PAGE = 50;
-
- static List<JsonNode> list(Client client, String apiEndpoint, TaskLogger logger) {
- List<JsonNode> result = new ArrayList<>();
- list(client, apiEndpoint, new PageDataConsumer() {
-
- @Override
- public void consume(List<JsonNode> pageData) {
- result.addAll(pageData);
- }
-
- }, logger);
- return result;
- }
-
- static void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer,
- TaskLogger logger) {
- URI uri;
- try {
- uri = new URIBuilder(apiEndpoint)
- .addParameter("pagelen", String.valueOf(PER_PAGE)).build();
- } catch (URISyntaxException e) {
- throw new RuntimeException(e);
- }
-
- while (true) {
- try {
- List<JsonNode> pageData = new ArrayList<>();
- JsonNode resultNode = JerseyUtils.get(client, uri.toString(), logger);
- if (resultNode.hasNonNull("values")) {
- for (JsonNode each: resultNode.get("values"))
- pageData.add(each);
- pageDataConsumer.consume(pageData);
- }
- if (resultNode.hasNonNull("next"))
- uri = new URIBuilder(resultNode.get("next").asText()).build();
- else
- break;
- } catch (URISyntaxException | InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
-}
\ No newline at end of file
diff --git a/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportWorkspace.java b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportWorkspace.java
new file mode 100644
index 0000000000..d841d12dd7
--- /dev/null
+++ b/server-plugin/server-plugin-import-bitbucketcloud/src/main/java/io/onedev/server/plugin/imports/bitbucketcloud/ImportWorkspace.java
@@ -0,0 +1,40 @@
+package io.onedev.server.plugin.imports.bitbucketcloud;
+
+import java.io.Serializable;
+import java.util.Map;
+
+import org.hibernate.validator.constraints.NotEmpty;
+
+import io.onedev.server.util.ComponentContext;
+import io.onedev.server.web.editable.BeanEditor;
+import io.onedev.server.web.editable.annotation.ChoiceProvider;
+import io.onedev.server.web.editable.annotation.Editable;
+
+@Editable
+public class ImportWorkspace implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ ImportServer server;
+
+ private String workspace;
+
+ @Editable(order=100, name="Bitbucket Workspace", description="Select workspace to import from")
+ @ChoiceProvider("getWorkspaceChoices")
+ @NotEmpty
+ public String getWorkspace() {
+ return workspace;
+ }
+
+ public void setWorkspace(String workspace) {
+ this.workspace = workspace;
+ }
+
+ @SuppressWarnings("unused")
+ private static Map<String, String> getWorkspaceChoices() {
+ BeanEditor editor = ComponentContext.get().getComponent().findParent(BeanEditor.class);
+ ImportWorkspace setting = (ImportWorkspace) editor.getModelObject();
+ return setting.server.listWorkspaces();
+ }
+
+}
diff --git a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaIssueImporter.java b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaIssueImporter.java
index 3f15c05387..69a63bc61c 100644
--- a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaIssueImporter.java
+++ b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaIssueImporter.java
@@ -1,6 +1,8 @@
package io.onedev.server.plugin.imports.gitea;
+import java.io.Serializable;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -10,35 +12,84 @@ import io.onedev.commons.utils.TaskLogger;
import io.onedev.server.imports.IssueImporter;
import io.onedev.server.model.Project;
import io.onedev.server.model.User;
-import io.onedev.server.web.util.WicketUtils;
+import io.onedev.server.web.util.ImportStep;
-public class GiteaIssueImporter extends IssueImporter<ImportServer, IssueImportSource, IssueImportOption> {
+public class GiteaIssueImporter implements IssueImporter {
private static final long serialVersionUID = 1L;
- @Override
- public String getName() {
- return ImportUtils.NAME;
- }
+ private final ImportStep<ImportServer> serverStep = new ImportStep<ImportServer>() {
- @Override
- public IssueImportSource getWhat(ImportServer where, TaskLogger logger) {
- WicketUtils.getPage().setMetaData(ImportServer.META_DATA_KEY, where);
- return new IssueImportSource();
- }
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Authenticate to Gitea";
+ }
+
+ @Override
+ protected ImportServer newSetting() {
+ return new ImportServer();
+ }
+
+ };
+
+ private final ImportStep<ImportRepository> repositoryStep = new ImportStep<ImportRepository>() {
+
+ private static final long serialVersionUID = 1L;
+ @Override
+ public String getTitle() {
+ return "Choose repository";
+ }
+
+ @Override
+ protected ImportRepository newSetting() {
+ ImportRepository repository = new ImportRepository();
+ repository.server = serverStep.getSetting();
+ return repository;
+ }
+
+ };
+
+ private final ImportStep<IssueImportOption> optionStep = new ImportStep<IssueImportOption>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Specify import option";
+ }
+
+ @Override
+ protected IssueImportOption newSetting() {
+ return serverStep.getSetting().buildIssueImportOption(
+ Lists.newArrayList(repositoryStep.getSetting().getRepository()));
+ }
+
+ };
+
@Override
- public IssueImportOption getHow(ImportServer where, IssueImportSource what, TaskLogger logger) {
- return ImportUtils.buildIssueImportOption(where, Lists.newArrayList(what.getRepository()), logger);
+ public String getName() {
+ return GiteaPluginModule.NAME;
}
@Override
- public String doImport(ImportServer where, IssueImportSource what, IssueImportOption how,
- Project project, boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {
- logger.log("Importing issues from repository " + what.getRepository() + "...");
+ public String doImport(Project project, boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {
+ ImportServer server = serverStep.getSetting();
+ String giteaRepo = repositoryStep.getSetting().getRepository();
+ IssueImportOption option = optionStep.getSetting();
+
+ logger.log("Importing issues from repository " + giteaRepo + "...");
Map<String, Optional<User>> users = new HashMap<>();
- return ImportUtils.importIssues(where, what.getRepository(), project, retainIssueNumbers, how, users, dryRun, logger)
+
+ return server.importIssues(giteaRepo, project, retainIssueNumbers, option, users, dryRun, logger)
.toHtml("Issues imported successfully");
}
+ @Override
+ public List<ImportStep<? extends Serializable>> getSteps() {
+ return Lists.newArrayList(serverStep, repositoryStep, optionStep);
+ }
+
}
diff --git a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaPluginModule.java b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaPluginModule.java
index 4e87a4224a..f060bdfb82 100644
--- a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaPluginModule.java
+++ b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaPluginModule.java
@@ -1,6 +1,5 @@
package io.onedev.server.plugin.imports.gitea;
-import java.io.Serializable;
import java.util.Collection;
import com.google.common.collect.Lists;
@@ -17,6 +16,8 @@ import io.onedev.server.imports.ProjectImporterContribution;
*/
public class GiteaPluginModule extends AbstractPluginModule {
+ static final String NAME = "Gitea";
+
@Override
protected void configure() {
super.configure();
@@ -25,7 +26,7 @@ public class GiteaPluginModule extends AbstractPluginModule {
contribute(ProjectImporterContribution.class, new ProjectImporterContribution() {
@Override
- public Collection<ProjectImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> getImporters() {
+ public Collection<ProjectImporter> getImporters() {
return Lists.newArrayList(new GiteaProjectImporter());
}
@@ -39,7 +40,7 @@ public class GiteaPluginModule extends AbstractPluginModule {
contribute(IssueImporterContribution.class, new IssueImporterContribution() {
@Override
- public Collection<IssueImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> getImporters() {
+ public Collection<IssueImporter> getImporters() {
return Lists.newArrayList(new GiteaIssueImporter());
}
diff --git a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaProjectImporter.java b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaProjectImporter.java
index 3646bce53e..42aa2f616d 100644
--- a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaProjectImporter.java
+++ b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/GiteaProjectImporter.java
@@ -1,141 +1,112 @@
package io.onedev.server.plugin.imports.gitea;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
+import java.io.Serializable;
import java.util.List;
-import java.util.Map;
-import java.util.Optional;
import java.util.stream.Collectors;
-import javax.ws.rs.client.Client;
-
-import org.apache.http.client.utils.URIBuilder;
-import org.joda.time.format.ISODateTimeFormat;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
import io.onedev.commons.utils.TaskLogger;
-import io.onedev.server.OneDev;
-import io.onedev.server.entitymanager.MilestoneManager;
-import io.onedev.server.entitymanager.ProjectManager;
import io.onedev.server.imports.ProjectImporter;
-import io.onedev.server.model.Milestone;
-import io.onedev.server.model.Project;
-import io.onedev.server.model.User;
-import io.onedev.server.storage.StorageManager;
-import io.onedev.server.util.JerseyUtils;
+import io.onedev.server.web.util.ImportStep;
-public class GiteaProjectImporter extends ProjectImporter<ImportServer, ProjectImportSource, ProjectImportOption> {
+public class GiteaProjectImporter implements ProjectImporter {
private static final long serialVersionUID = 1L;
+
+ private final ImportStep<ImportServer> serverStep = new ImportStep<ImportServer>() {
- @Override
- public String getName() {
- return ImportUtils.NAME;
- }
+ private static final long serialVersionUID = 1L;
- @Override
- public ProjectImportSource getWhat(ImportServer where, TaskLogger logger) {
- ProjectImportSource importSource = new ProjectImportSource();
- Client client = where.newClient();
- try {
- String apiEndpoint = where.getApiEndpoint("/user/repos");
- for (JsonNode repoNode: ImportUtils.list(client, apiEndpoint, logger)) {
- String repoName = repoNode.get("name").asText();
- String ownerName = repoNode.get("owner").get("login").asText();
+ @Override
+ public String getTitle() {
+ return "Authenticate to Gitea";
+ }
+
+ @Override
+ protected ImportServer newSetting() {
+ return new ImportServer();
+ }
+
+ };
+
+ private final ImportStep<ImportOrganization> organizationStep = new ImportStep<ImportOrganization>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Choose organization";
+ }
+
+ @Override
+ protected ImportOrganization newSetting() {
+ ImportOrganization organization = new ImportOrganization();
+ organization.server = serverStep.getSetting();
+ return organization;
+ }
+
+ };
+
+ private final ImportStep<ImportRepositories> repositoriesStep = new ImportStep<ImportRepositories>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Specify repositories";
+ }
+
+ @Override
+ protected ImportRepositories newSetting() {
+ ImportRepositories repositories = new ImportRepositories();
+ String organization = organizationStep.getSetting().getOrganization();
+ for (String repository: serverStep.getSetting().listRepositories(organization)) {
ProjectMapping projectMapping = new ProjectMapping();
- projectMapping.setGiteaRepo(ownerName + "/" + repoName);
- projectMapping.setOneDevProject(ownerName + "/" + repoName);
- importSource.getProjectMappings().add(projectMapping);
- }
- } finally {
- client.close();
+ projectMapping.setGiteaRepo(repository);
+ projectMapping.setOneDevProject(repository);
+ repositories.getProjectMappings().add(projectMapping);
+ }
+ return repositories;
}
- return importSource;
+
+ };
+
+ private final ImportStep<ProjectImportOption> optionStep = new ImportStep<ProjectImportOption>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Specify import option";
+ }
+
+ @Override
+ protected ProjectImportOption newSetting() {
+ ProjectImportOption option = new ProjectImportOption();
+ List<String> giteaRepos = repositoriesStep.getSetting().getProjectMappings().stream()
+ .map(it->it.getGiteaRepo()).collect(Collectors.toList());
+ option.setIssueImportOption(serverStep.getSetting().buildIssueImportOption(giteaRepos));
+ return option;
+ }
+
+ };
+
+ @Override
+ public String getName() {
+ return GiteaPluginModule.NAME;
}
@Override
- public ProjectImportOption getHow(ImportServer where, ProjectImportSource what, TaskLogger logger) {
- ProjectImportOption importOption = new ProjectImportOption();
- List<String> repos = what.getProjectMappings().stream().map(it->it.getGiteaRepo()).collect(Collectors.toList());
- importOption.setIssueImportOption(ImportUtils.buildIssueImportOption(where, repos, logger));
- return importOption;
+ public String doImport(boolean dryRun, TaskLogger logger) {
+ ImportRepositories repositories = repositoriesStep.getSetting();
+ ProjectImportOption option = optionStep.getSetting();
+ return serverStep.getSetting().importProjects(repositories, option, dryRun, logger);
}
@Override
- public String doImport(ImportServer where, ProjectImportSource what, ProjectImportOption how,
- boolean dryRun, TaskLogger logger) {
- Collection<Long> projectIds = new ArrayList<>();
- Client client = where.newClient();
- try {
- Map<String, Optional<User>> users = new HashMap<>();
- ImportResult result = new ImportResult();
- for (ProjectMapping projectMapping: what.getProjectMappings()) {
- logger.log("Cloning code from repository " + projectMapping.getGiteaRepo() + "...");
-
- String apiEndpoint = where.getApiEndpoint("/repos/" + projectMapping.getGiteaRepo());
- JsonNode repoNode = JerseyUtils.get(client, apiEndpoint, logger);
- ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
- Project project = projectManager.initialize(projectMapping.getOneDevProject());
- Preconditions.checkState(project.isNew());
- project.setDescription(repoNode.get("description").asText(null));
- project.setIssueManagement(repoNode.get("has_issues").asBoolean());
-
- boolean isPrivate = repoNode.get("private").asBoolean();
- if (!isPrivate && how.getPublicRole() != null)
- project.setDefaultRole(how.getPublicRole());
-
- URIBuilder builder = new URIBuilder(repoNode.get("clone_url").asText());
- if (isPrivate)
- builder.setUserInfo("git", where.getAccessToken());
-
- if (!dryRun) {
- projectManager.clone(project, builder.build().toString());
- projectIds.add(project.getId());
- }
-
- if (how.getIssueImportOption() != null) {
- List<Milestone> milestones = new ArrayList<>();
- logger.log("Importing milestones from repository " + projectMapping.getGiteaRepo() + "...");
- apiEndpoint = where.getApiEndpoint("/repos/" + projectMapping.getGiteaRepo() + "/milestones?state=all");
- for (JsonNode milestoneNode: ImportUtils.list(client, apiEndpoint, logger)) {
- Milestone milestone = new Milestone();
- milestone.setName(milestoneNode.get("title").asText());
- milestone.setDescription(milestoneNode.get("description").asText(null));
- milestone.setProject(project);
- String dueDateString = milestoneNode.get("due_on").asText(null);
- if (dueDateString != null)
- milestone.setDueDate(ISODateTimeFormat.dateTimeNoMillis().parseDateTime(dueDateString).toDate());
- if (milestoneNode.get("state").asText().equals("closed"))
- milestone.setClosed(true);
-
- milestones.add(milestone);
- project.getMilestones().add(milestone);
-
- if (!dryRun)
- OneDev.getInstance(MilestoneManager.class).save(milestone);
- }
-
- logger.log("Importing issues from repository " + projectMapping.getGiteaRepo() + "...");
- ImportResult currentResult = ImportUtils.importIssues(where, projectMapping.getGiteaRepo(),
- project, true, how.getIssueImportOption(), users, dryRun, logger);
- result.nonExistentLogins.addAll(currentResult.nonExistentLogins);
- result.nonExistentMilestones.addAll(currentResult.nonExistentMilestones);
- result.unmappedIssueLabels.addAll(currentResult.unmappedIssueLabels);
- result.issuesImported |= currentResult.issuesImported;
- }
- }
-
- return result.toHtml("Repositories imported successfully");
- } catch (Exception e) {
- for (Long projectId: projectIds)
- OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);
- throw new RuntimeException(e);
- } finally {
- client.close();
- }
+ public List<ImportStep<? extends Serializable>> getSteps() {
+ return Lists.newArrayList(serverStep, organizationStep, repositoriesStep, optionStep);
}
}
diff --git a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportOrganization.java b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportOrganization.java
new file mode 100644
index 0000000000..3f754db584
--- /dev/null
+++ b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportOrganization.java
@@ -0,0 +1,38 @@
+package io.onedev.server.plugin.imports.gitea;
+
+import java.io.Serializable;
+import java.util.List;
+
+import io.onedev.server.util.ComponentContext;
+import io.onedev.server.web.editable.BeanEditor;
+import io.onedev.server.web.editable.annotation.ChoiceProvider;
+import io.onedev.server.web.editable.annotation.Editable;
+
+@Editable
+public class ImportOrganization implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ ImportServer server;
+
+ private String organization;
+
+ @Editable(order=100, name="Gitea Organization", description="Select organization to import from. "
+ + "Leave empty to import from repositories under current account")
+ @ChoiceProvider("getOrganizationChoices")
+ public String getOrganization() {
+ return organization;
+ }
+
+ public void setOrganization(String organization) {
+ this.organization = organization;
+ }
+
+ @SuppressWarnings("unused")
+ private static List<String> getOrganizationChoices() {
+ BeanEditor editor = ComponentContext.get().getComponent().findParent(BeanEditor.class);
+ ImportOrganization setting = (ImportOrganization) editor.getModelObject();
+ return setting.server.listOrganizations();
+ }
+
+}
diff --git a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ProjectImportSource.java b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportRepositories.java
similarity index 94%
rename from server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ProjectImportSource.java
rename to server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportRepositories.java
index 0ceea04233..f980982743 100644
--- a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ProjectImportSource.java
+++ b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportRepositories.java
@@ -15,10 +15,11 @@ import io.onedev.server.model.Project;
import io.onedev.server.util.validation.Validatable;
import io.onedev.server.util.validation.annotation.ClassValidating;
import io.onedev.server.web.editable.annotation.Editable;
+import io.onedev.server.web.editable.annotation.OmitName;
@Editable
@ClassValidating
-public class ProjectImportSource implements Serializable, Validatable {
+public class ImportRepositories implements Serializable, Validatable {
private static final long serialVersionUID = 1L;
@@ -26,6 +27,7 @@ public class ProjectImportSource implements Serializable, Validatable {
@Editable(order=100, name="Repositories to Import")
@Size(min=1, max=10000, message="No repositories to import")
+ @OmitName
public List<ProjectMapping> getProjectMappings() {
return projectMappings;
}
diff --git a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportRepository.java b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportRepository.java
new file mode 100644
index 0000000000..2e7d46ae71
--- /dev/null
+++ b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportRepository.java
@@ -0,0 +1,39 @@
+package io.onedev.server.plugin.imports.gitea;
+
+import java.util.List;
+
+import org.hibernate.validator.constraints.NotEmpty;
+
+import io.onedev.server.util.ComponentContext;
+import io.onedev.server.util.EditContext;
+import io.onedev.server.web.editable.BeanEditor;
+import io.onedev.server.web.editable.annotation.ChoiceProvider;
+import io.onedev.server.web.editable.annotation.Editable;
+
+@Editable
+public class ImportRepository extends ImportOrganization {
+
+ private static final long serialVersionUID = 1L;
+
+ private String repository;
+
+ @Editable(order=200, name="Gitea Repository", description="Select repository to import from")
+ @ChoiceProvider("getRepositoryChoices")
+ @NotEmpty
+ public String getRepository() {
+ return repository;
+ }
+
+ public void setRepository(String repository) {
+ this.repository = repository;
+ }
+
+ @SuppressWarnings("unused")
+ private static List<String> getRepositoryChoices() {
+ BeanEditor editor = ComponentContext.get().getComponent().findParent(BeanEditor.class);
+ ImportRepository setting = (ImportRepository) editor.getModelObject();
+ String organization = (String) EditContext.get().getInputValue("organization");
+ return setting.server.listRepositories(organization);
+ }
+
+}
diff --git a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportServer.java b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportServer.java
index 29b5266fcd..86ca901faa 100644
--- a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportServer.java
+++ b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportServer.java
@@ -1,7 +1,24 @@
package io.onedev.server.plugin.imports.gitea;
import java.io.Serializable;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.time.Instant;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.annotation.Nullable;
import javax.validation.ConstraintValidatorContext;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
@@ -9,13 +26,46 @@ import javax.ws.rs.client.Invocation;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response;
-import org.apache.wicket.MetaDataKey;
+import org.apache.http.client.utils.URIBuilder;
import org.glassfish.jersey.client.ClientProperties;
import org.glassfish.jersey.client.oauth2.OAuth2ClientSupport;
import org.hibernate.validator.constraints.NotEmpty;
+import org.joda.time.format.ISODateTimeFormat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.unbescape.html.HtmlEscape;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.base.Preconditions;
+
+import edu.emory.mathcs.backport.java.util.Collections;
+import io.onedev.commons.utils.ExplicitException;
import io.onedev.commons.utils.StringUtils;
+import io.onedev.commons.utils.TaskLogger;
+import io.onedev.server.OneDev;
+import io.onedev.server.entitymanager.IssueManager;
+import io.onedev.server.entitymanager.MilestoneManager;
+import io.onedev.server.entitymanager.ProjectManager;
+import io.onedev.server.entitymanager.SettingManager;
+import io.onedev.server.entitymanager.UserManager;
+import io.onedev.server.entityreference.ReferenceMigrator;
+import io.onedev.server.model.Issue;
+import io.onedev.server.model.IssueComment;
+import io.onedev.server.model.IssueField;
+import io.onedev.server.model.IssueSchedule;
+import io.onedev.server.model.Milestone;
+import io.onedev.server.model.Project;
+import io.onedev.server.model.User;
+import io.onedev.server.model.support.LastUpdate;
+import io.onedev.server.model.support.administration.GlobalIssueSetting;
+import io.onedev.server.model.support.inputspec.InputSpec;
+import io.onedev.server.model.support.issue.field.spec.FieldSpec;
+import io.onedev.server.persistence.dao.Dao;
+import io.onedev.server.storage.StorageManager;
+import io.onedev.server.util.DateUtils;
import io.onedev.server.util.JerseyUtils;
+import io.onedev.server.util.JerseyUtils.PageDataConsumer;
+import io.onedev.server.util.Pair;
import io.onedev.server.util.validation.Validatable;
import io.onedev.server.util.validation.annotation.ClassValidating;
import io.onedev.server.web.editable.annotation.Editable;
@@ -27,15 +77,13 @@ public class ImportServer implements Serializable, Validatable {
private static final long serialVersionUID = 1L;
- static final MetaDataKey<ImportServer> META_DATA_KEY = new MetaDataKey<ImportServer>() {
-
- private static final long serialVersionUID = 1L;
-
- };
-
- static final String PROP_API_URL = "apiUrl";
+ private static final Logger logger = LoggerFactory.getLogger(ImportServer.class);
- static final String PROP_ACCESS_TOKEN = "accessToken";
+ private static final int PER_PAGE = 50;
+
+ private static final String PROP_API_URL = "apiUrl";
+
+ private static final String PROP_ACCESS_TOKEN = "accessToken";
private String apiUrl;
@@ -62,10 +110,502 @@ public class ImportServer implements Serializable, Validatable {
this.accessToken = accessToken;
}
- public String getApiEndpoint(String apiPath) {
+ private String getApiEndpoint(String apiPath) {
return StringUtils.stripEnd(apiUrl, "/") + "/" + StringUtils.stripStart(apiPath, "/");
}
+ List<String> listOrganizations() {
+ List<String> organizations = new ArrayList<>();
+
+ Client client = newClient();
+ try {
+ String apiEndpoint = getApiEndpoint("/user/orgs");
+ for (JsonNode orgNode: list(client, apiEndpoint, new TaskLogger() {
+
+ @Override
+ public void log(String message, String sessionId) {
+ logger.info(message);
+ }
+
+ })) {
+ organizations.add(orgNode.get("username").asText());
+ }
+ Collections.sort(organizations);
+ } catch (Exception e) {
+ logger.error("Error listing organizations", e);
+ } finally {
+ client.close();
+ }
+
+ return organizations;
+ }
+
+ List<String> listRepositories(@Nullable String organization) {
+ Client client = newClient();
+ try {
+ String apiEndpoint;
+ if (organization != null)
+ apiEndpoint = getApiEndpoint("/orgs/" + organization + "/repos");
+ else
+ apiEndpoint = getApiEndpoint("/user/repos");
+ List<String> repositories = new ArrayList<>();
+ for (JsonNode repoNode: list(client, apiEndpoint, new TaskLogger() {
+
+ @Override
+ public void log(String message, String sessionId) {
+ logger.info(message);
+ }
+
+ })) {
+ String repoName = repoNode.get("name").asText();
+ JsonNode ownerNode = repoNode.get("owner");
+ String ownerName = ownerNode.get("login").asText();
+ JsonNode emailNode = ownerNode.get("email");
+ if (organization != null || emailNode != null && StringUtils.isNotBlank(emailNode.asText()))
+ repositories.add(ownerName + "/" + repoName);
+ }
+ Collections.sort(repositories);
+ return repositories;
+ } finally {
+ client.close();
+ }
+
+ }
+
+ IssueImportOption buildIssueImportOption(Collection<String> repositories) {
+ IssueImportOption option = new IssueImportOption();
+ Client client = newClient();
+ try {
+ TaskLogger taskLogger = new TaskLogger() {
+
+ @Override
+ public void log(String message, String sessionId) {
+ logger.info(message);
+ }
+
+ };
+ Set<String> labels = new LinkedHashSet<>();
+ for (String repo: repositories) {
+ String apiEndpoint = getApiEndpoint("/repos/" + repo + "/labels");
+ for (JsonNode labelNode: list(client, apiEndpoint, taskLogger))
+ labels.add(labelNode.get("name").asText());
+ try {
+ apiEndpoint = getApiEndpoint("/orgs/" + StringUtils.substringBefore(repo, "/") + "/labels");
+ for (JsonNode labelNode: list(client, apiEndpoint, taskLogger))
+ labels.add(labelNode.get("name").asText());
+ } catch (Exception e) {
+ // ignore as exception might be thrown if repo belongs to a user account
+ }
+ }
+
+ for (String label: labels) {
+ IssueLabelMapping mapping = new IssueLabelMapping();
+ mapping.setGiteaIssueLabel(label);
+ option.getIssueLabelMappings().add(mapping);
+ }
+ } finally {
+ client.close();
+ }
+ return option;
+ }
+
+ @Nullable
+ private User getUser(Map<String, Optional<User>> users, JsonNode userNode, TaskLogger logger) {
+ String login = userNode.get("login").asText();
+ Optional<User> userOpt = users.get(login);
+ if (userOpt == null) {
+ String email = null;
+ if (userNode.hasNonNull("email"))
+ email = userNode.get("email").asText(null);
+ if (email != null)
+ userOpt = Optional.ofNullable(OneDev.getInstance(UserManager.class).findByVerifiedEmailAddress(email));
+ else
+ userOpt = Optional.empty();
+ users.put(login, userOpt);
+ }
+ return userOpt.orElse(null);
+ }
+
+ ImportResult importIssues(String giteaRepo, Project oneDevProject, boolean useExistingIssueNumbers,
+ IssueImportOption option, Map<String, Optional<User>> users, boolean dryRun, TaskLogger logger) {
+ Client client = newClient();
+ try {
+ Set<String> nonExistentMilestones = new HashSet<>();
+ Set<String> nonExistentLogins = new HashSet<>();
+ Set<String> unmappedIssueLabels = new HashSet<>();
+
+ Map<String, Pair<FieldSpec, String>> labelMappings = new HashMap<>();
+ Map<String, Milestone> milestoneMappings = new HashMap<>();
+
+ for (IssueLabelMapping mapping: option.getIssueLabelMappings()) {
+ String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");
+ String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");
+ FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);
+ if (fieldSpec == null)
+ throw new ExplicitException("No field spec found: " + oneDevFieldName);
+ labelMappings.put(mapping.getGiteaIssueLabel(), new Pair<>(fieldSpec, oneDevFieldValue));
+ }
+
+ for (Milestone milestone: oneDevProject.getMilestones())
+ milestoneMappings.put(milestone.getName(), milestone);
+
+ String initialIssueState = getIssueSetting().getInitialStateSpec().getName();
+
+ List<Issue> issues = new ArrayList<>();
+
+ Map<Long, Long> issueNumberMappings = new HashMap<>();
+
+ AtomicInteger numOfImportedIssues = new AtomicInteger(0);
+ PageDataConsumer pageDataConsumer = new PageDataConsumer() {
+
+ private String joinAsMultilineHtml(List<String> values) {
+ List<String> escapedValues = new ArrayList<>();
+ for (String value: values)
+ escapedValues.add(HtmlEscape.escapeHtml5(value));
+ return StringUtils.join(escapedValues, "<br>");
+ }
+
+ @Override
+ public void consume(List<JsonNode> pageData) throws InterruptedException {
+ for (JsonNode issueNode: pageData) {
+ if (Thread.interrupted())
+ throw new InterruptedException();
+
+ Map<String, String> extraIssueInfo = new LinkedHashMap<>();
+
+ Issue issue = new Issue();
+ issue.setProject(oneDevProject);
+ issue.setTitle(issueNode.get("title").asText());
+ issue.setDescription(issueNode.get("body").asText(null));
+
+ issue.setNumberScope(oneDevProject.getForkRoot());
+
+ Long oldNumber = issueNode.get("number").asLong();
+ Long newNumber;
+ if (dryRun || useExistingIssueNumbers)
+ newNumber = oldNumber;
+ else
+ newNumber = OneDev.getInstance(IssueManager.class).getNextNumber(oneDevProject);
+ issue.setNumber(newNumber);
+ issueNumberMappings.put(oldNumber, newNumber);
+
+ String issueFQN = giteaRepo + "#" + oldNumber;
+
+ if (issueNode.get("state").asText().equals("closed"))
+ issue.setState(option.getClosedIssueState());
+ else
+ issue.setState(initialIssueState);
+
+ if (issueNode.hasNonNull("milestone")) {
+ String milestoneName = issueNode.get("milestone").get("title").asText();
+ Milestone milestone = milestoneMappings.get(milestoneName);
+ if (milestone != null) {
+ IssueSchedule schedule = new IssueSchedule();
+ schedule.setIssue(issue);
+ schedule.setMilestone(milestone);
+ issue.getSchedules().add(schedule);
+ } else {
+ extraIssueInfo.put("Milestone", milestoneName);
+ nonExistentMilestones.add(milestoneName);
+ }
+ }
+
+ JsonNode userNode = issueNode.get("user");
+ User user = getUser(users, userNode, logger);
+ if (user != null) {
+ issue.setSubmitter(user);
+ } else {
+ issue.setSubmitter(OneDev.getInstance(UserManager.class).getUnknown());
+ nonExistentLogins.add(userNode.get("login").asText());
+ }
+
+ String created_at = issueNode.get("created_at").asText();
+ issue.setSubmitDate(Date.from(Instant.from(
+ DateTimeFormatter.ISO_DATE_TIME.parse(created_at))));
+
+ LastUpdate lastUpdate = new LastUpdate();
+ lastUpdate.setActivity("Opened");
+ lastUpdate.setDate(issue.getSubmitDate());
+ lastUpdate.setUser(issue.getSubmitter());
+ issue.setLastUpdate(lastUpdate);
+
+ List<JsonNode> assigneeNodes = new ArrayList<>();
+ if (issueNode.hasNonNull("assignees")) {
+ for (JsonNode assigneeNode: issueNode.get("assignees"))
+ assigneeNodes.add(assigneeNode);
+ } else if (issueNode.hasNonNull("assignee")) {
+ assigneeNodes.add(issueNode.get("assignee"));
+ }
+
+ for (JsonNode assigneeNode: assigneeNodes) {
+ IssueField assigneeField = new IssueField();
+ assigneeField.setIssue(issue);
+ assigneeField.setName(option.getAssigneesIssueField());
+ assigneeField.setType(InputSpec.USER);
+
+ user = getUser(users, assigneeNode, logger);
+ if (user != null) {
+ assigneeField.setValue(user.getName());
+ issue.getFields().add(assigneeField);
+ } else {
+ nonExistentLogins.add(assigneeNode.get("login").asText());
+ }
+ }
+
+ if (option.getDueDateIssueField() != null) {
+ String dueDate = issueNode.get("due_date").asText(null);
+ if (dueDate != null) {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(option.getDueDateIssueField());
+ issueField.setType(InputSpec.DATE);
+ issueField.setValue(DateUtils.formatDate(Date.from(
+ Instant.from(DateTimeFormatter.ISO_DATE_TIME.parse(dueDate)))));
+ issue.getFields().add(issueField);
+ }
+ }
+
+ List<String> currentUnmappedLabels = new ArrayList<>();
+ for (JsonNode labelNode: issueNode.get("labels")) {
+ String labelName = labelNode.get("name").asText();
+ Pair<FieldSpec, String> mapped = labelMappings.get(labelName);
+ if (mapped != null) {
+ IssueField labelField = new IssueField();
+ labelField.setIssue(issue);
+ labelField.setName(mapped.getFirst().getName());
+ labelField.setType(InputSpec.ENUMERATION);
+ labelField.setValue(mapped.getSecond());
+ labelField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));
+ issue.getFields().add(labelField);
+ } else {
+ currentUnmappedLabels.add(labelName);
+ unmappedIssueLabels.add(HtmlEscape.escapeHtml5(labelName));
+ }
+ }
+
+ if (!currentUnmappedLabels.isEmpty())
+ extraIssueInfo.put("Labels", joinAsMultilineHtml(currentUnmappedLabels));
+
+ String apiEndpoint = getApiEndpoint("/repos/" + giteaRepo
+ + "/issues/" + oldNumber + "/comments");
+ for (JsonNode commentNode: list(client, apiEndpoint, logger)) {
+ String commentContent = commentNode.get("body").asText(null);
+ if (commentContent != null) {
+ IssueComment comment = new IssueComment();
+ comment.setIssue(issue);
+ comment.setContent(commentContent);
+
+ created_at = commentNode.get("created_at").asText();
+ comment.setDate(Date.from(Instant.from(
+ DateTimeFormatter.ISO_DATE_TIME.parse(created_at))));
+
+ userNode = commentNode.get("user");
+ user = getUser(users, userNode, logger);
+ if (user != null) {
+ comment.setUser(user);
+ } else {
+ comment.setUser(OneDev.getInstance(UserManager.class).getUnknown());
+ nonExistentLogins.add(userNode.get("username").asText());
+ }
+ issue.getComments().add(comment);
+ }
+ }
+
+ issue.setCommentCount(issue.getComments().size());
+
+ Set<String> fieldAndValues = new HashSet<>();
+ for (IssueField field: issue.getFields()) {
+ String fieldAndValue = field.getName() + "::" + field.getValue();
+ if (!fieldAndValues.add(fieldAndValue)) {
+ String errorMessage = String.format(
+ "Duplicate issue field mapping (issue: %s, field: %s)",
+ issueFQN, fieldAndValue);
+ throw new ExplicitException(errorMessage);
+ }
+ }
+
+ if (!extraIssueInfo.isEmpty()) {
+ StringBuilder builder = new StringBuilder("|");
+ for (String key: extraIssueInfo.keySet())
+ builder.append(key).append("|");
+ builder.append("\n|");
+ extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));
+ builder.append("\n|");
+ for (String value: extraIssueInfo.values())
+ builder.append(value).append("|");
+
+ if (issue.getDescription() != null)
+ issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());
+ else
+ issue.setDescription(builder.toString());
+ }
+ issues.add(issue);
+ }
+ logger.log("Imported " + numOfImportedIssues.addAndGet(pageData.size()) + " issues");
+ }
+
+ };
+
+ String apiEndpoint = getApiEndpoint("/repos/" + giteaRepo + "/issues?state=all&type=issues");
+ list(client, apiEndpoint, pageDataConsumer, logger);
+
+ if (!dryRun) {
+ ReferenceMigrator migrator = new ReferenceMigrator(Issue.class, issueNumberMappings);
+ Dao dao = OneDev.getInstance(Dao.class);
+ for (Issue issue: issues) {
+ if (issue.getDescription() != null)
+ issue.setDescription(migrator.migratePrefixed(issue.getDescription(), "#"));
+
+ OneDev.getInstance(IssueManager.class).save(issue);
+ for (IssueSchedule schedule: issue.getSchedules())
+ dao.persist(schedule);
+ for (IssueField field: issue.getFields())
+ dao.persist(field);
+ for (IssueComment comment: issue.getComments()) {
+ comment.setContent(migrator.migratePrefixed(comment.getContent(), "#"));
+ dao.persist(comment);
+ }
+ }
+ }
+
+ ImportResult result = new ImportResult();
+ result.nonExistentLogins.addAll(nonExistentLogins);
+ result.nonExistentMilestones.addAll(nonExistentMilestones);
+ result.unmappedIssueLabels.addAll(unmappedIssueLabels);
+
+ if (numOfImportedIssues.get() != 0)
+ result.issuesImported = true;
+
+ return result;
+ } finally {
+ client.close();
+ }
+ }
+
+ String importProjects(ImportRepositories repositories, ProjectImportOption option, boolean dryRun, TaskLogger logger) {
+ Collection<Long> projectIds = new ArrayList<>();
+ Client client = newClient();
+ try {
+ Map<String, Optional<User>> users = new HashMap<>();
+ ImportResult result = new ImportResult();
+ for (ProjectMapping projectMapping: repositories.getProjectMappings()) {
+ logger.log("Cloning code from repository " + projectMapping.getGiteaRepo() + "...");
+
+ String apiEndpoint = getApiEndpoint("/repos/" + projectMapping.getGiteaRepo());
+ JsonNode repoNode = JerseyUtils.get(client, apiEndpoint, logger);
+ ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
+ Project project = projectManager.initialize(projectMapping.getOneDevProject());
+ Preconditions.checkState(project.isNew());
+ project.setDescription(repoNode.get("description").asText(null));
+ project.setIssueManagement(repoNode.get("has_issues").asBoolean());
+
+ boolean isPrivate = repoNode.get("private").asBoolean();
+ if (!isPrivate && option.getPublicRole() != null)
+ project.setDefaultRole(option.getPublicRole());
+
+ URIBuilder builder = new URIBuilder(repoNode.get("clone_url").asText());
+ if (isPrivate)
+ builder.setUserInfo("git", getAccessToken());
+
+ if (!dryRun) {
+ projectManager.clone(project, builder.build().toString());
+ projectIds.add(project.getId());
+ }
+
+ if (option.getIssueImportOption() != null) {
+ List<Milestone> milestones = new ArrayList<>();
+ logger.log("Importing milestones from repository " + projectMapping.getGiteaRepo() + "...");
+ apiEndpoint = getApiEndpoint("/repos/" + projectMapping.getGiteaRepo() + "/milestones?state=all");
+ for (JsonNode milestoneNode: list(client, apiEndpoint, logger)) {
+ Milestone milestone = new Milestone();
+ milestone.setName(milestoneNode.get("title").asText());
+ milestone.setDescription(milestoneNode.get("description").asText(null));
+ milestone.setProject(project);
+ String dueDateString = milestoneNode.get("due_on").asText(null);
+ if (dueDateString != null)
+ milestone.setDueDate(ISODateTimeFormat.dateTimeNoMillis().parseDateTime(dueDateString).toDate());
+ if (milestoneNode.get("state").asText().equals("closed"))
+ milestone.setClosed(true);
+
+ milestones.add(milestone);
+ project.getMilestones().add(milestone);
+
+ if (!dryRun)
+ OneDev.getInstance(MilestoneManager.class).save(milestone);
+ }
+
+ logger.log("Importing issues from repository " + projectMapping.getGiteaRepo() + "...");
+ ImportResult currentResult = importIssues(projectMapping.getGiteaRepo(),
+ project, true, option.getIssueImportOption(), users, dryRun, logger);
+ result.nonExistentLogins.addAll(currentResult.nonExistentLogins);
+ result.nonExistentMilestones.addAll(currentResult.nonExistentMilestones);
+ result.unmappedIssueLabels.addAll(currentResult.unmappedIssueLabels);
+ result.issuesImported |= currentResult.issuesImported;
+ }
+ }
+
+ return result.toHtml("Repositories imported successfully");
+ } catch (Exception e) {
+ for (Long projectId: projectIds)
+ OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);
+ throw new RuntimeException(e);
+ } finally {
+ client.close();
+ }
+ }
+
+ private GlobalIssueSetting getIssueSetting() {
+ return OneDev.getInstance(SettingManager.class).getIssueSetting();
+ }
+
+ private List<JsonNode> list(Client client, String apiEndpoint, TaskLogger logger) {
+ List<JsonNode> result = new ArrayList<>();
+ list(client, apiEndpoint, new PageDataConsumer() {
+
+ @Override
+ public void consume(List<JsonNode> pageData) {
+ result.addAll(pageData);
+ }
+
+ }, logger);
+ return result;
+ }
+
+ private void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer,
+ TaskLogger logger) {
+ URI uri;
+ try {
+ uri = new URIBuilder(apiEndpoint)
+ .addParameter("limit", String.valueOf(PER_PAGE)).build();
+ } catch (URISyntaxException e) {
+ throw new RuntimeException(e);
+ }
+
+ int page = 1;
+ while (true) {
+ try {
+ URIBuilder builder = new URIBuilder(uri);
+ builder.addParameter("page", String.valueOf(page));
+ List<JsonNode> pageData = new ArrayList<>();
+ for (JsonNode each: JerseyUtils.get(client, builder.build().toString(), logger))
+ pageData.add(each);
+ pageDataConsumer.consume(pageData);
+ if (pageData.size() < PER_PAGE)
+ break;
+ page++;
+ } catch (URISyntaxException|InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private Client newClient() {
+ Client client = ClientBuilder.newClient();
+ client.property(ClientProperties.FOLLOW_REDIRECTS, true);
+ client.register(OAuth2ClientSupport.feature(getAccessToken()));
+ return client;
+ }
+
@Override
public boolean isValid(ConstraintValidatorContext context) {
Client client = ClientBuilder.newClient();
@@ -111,11 +651,4 @@ public class ImportServer implements Serializable, Validatable {
return true;
}
- Client newClient() {
- Client client = ClientBuilder.newClient();
- client.property(ClientProperties.FOLLOW_REDIRECTS, true);
- client.register(OAuth2ClientSupport.feature(getAccessToken()));
- return client;
- }
-
}
diff --git a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportUtils.java b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportUtils.java
deleted file mode 100644
index 537a0b4bde..0000000000
--- a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/ImportUtils.java
+++ /dev/null
@@ -1,407 +0,0 @@
-package io.onedev.server.plugin.imports.gitea;
-
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.time.Instant;
-import java.time.format.DateTimeFormatter;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import javax.annotation.Nullable;
-import javax.ws.rs.client.Client;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.http.client.utils.URIBuilder;
-import org.unbescape.html.HtmlEscape;
-
-import com.fasterxml.jackson.databind.JsonNode;
-
-import io.onedev.commons.utils.ExplicitException;
-import io.onedev.commons.utils.TaskLogger;
-import io.onedev.server.OneDev;
-import io.onedev.server.entitymanager.IssueManager;
-import io.onedev.server.entitymanager.SettingManager;
-import io.onedev.server.entitymanager.UserManager;
-import io.onedev.server.entityreference.ReferenceMigrator;
-import io.onedev.server.model.Issue;
-import io.onedev.server.model.IssueComment;
-import io.onedev.server.model.IssueField;
-import io.onedev.server.model.IssueSchedule;
-import io.onedev.server.model.Milestone;
-import io.onedev.server.model.Project;
-import io.onedev.server.model.User;
-import io.onedev.server.model.support.LastUpdate;
-import io.onedev.server.model.support.administration.GlobalIssueSetting;
-import io.onedev.server.model.support.inputspec.InputSpec;
-import io.onedev.server.model.support.issue.field.spec.FieldSpec;
-import io.onedev.server.persistence.dao.Dao;
-import io.onedev.server.util.DateUtils;
-import io.onedev.server.util.JerseyUtils;
-import io.onedev.server.util.JerseyUtils.PageDataConsumer;
-import io.onedev.server.util.Pair;
-
-public class ImportUtils {
-
- public static final String NAME = "Gitea";
-
- static final int PER_PAGE = 50;
-
- static IssueImportOption buildIssueImportOption(ImportServer server, Collection<String> repositories, TaskLogger logger) {
- IssueImportOption importOption = new IssueImportOption();
- Client client = server.newClient();
- try {
- Set<String> labels = new LinkedHashSet<>();
- for (String repo: repositories) {
- String apiEndpoint = server.getApiEndpoint("/repos/" + repo + "/labels");
- for (JsonNode labelNode: list(client, apiEndpoint, logger))
- labels.add(labelNode.get("name").asText());
- try {
- apiEndpoint = server.getApiEndpoint("/orgs/" + StringUtils.substringBefore(repo, "/") + "/labels");
- for (JsonNode labelNode: list(client, apiEndpoint, logger))
- labels.add(labelNode.get("name").asText());
- } catch (Exception e) {
- // ignore as exception might be thrown if repo belongs to a user account
- }
- }
-
- for (String label: labels) {
- IssueLabelMapping mapping = new IssueLabelMapping();
- mapping.setGiteaIssueLabel(label);
- importOption.getIssueLabelMappings().add(mapping);
- }
- } finally {
- client.close();
- }
- return importOption;
- }
-
- @Nullable
- static User getUser(Map<String, Optional<User>> users, JsonNode userNode, TaskLogger logger) {
- String login = userNode.get("login").asText();
- Optional<User> userOpt = users.get(login);
- if (userOpt == null) {
- String email = null;
- if (userNode.hasNonNull("email"))
- email = userNode.get("email").asText(null);
- if (email != null)
- userOpt = Optional.ofNullable(OneDev.getInstance(UserManager.class).findByVerifiedEmailAddress(email));
- else
- userOpt = Optional.empty();
- users.put(login, userOpt);
- }
- return userOpt.orElse(null);
- }
-
- static ImportResult importIssues(ImportServer server, String giteaRepo, Project oneDevProject,
- boolean useExistingIssueNumbers, IssueImportOption importOption, Map<String, Optional<User>> users,
- boolean dryRun, TaskLogger logger) {
- Client client = server.newClient();
- try {
- Set<String> nonExistentMilestones = new HashSet<>();
- Set<String> nonExistentLogins = new HashSet<>();
- Set<String> unmappedIssueLabels = new HashSet<>();
-
- Map<String, Pair<FieldSpec, String>> labelMappings = new HashMap<>();
- Map<String, Milestone> milestoneMappings = new HashMap<>();
-
- for (IssueLabelMapping mapping: importOption.getIssueLabelMappings()) {
- String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");
- String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");
- FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);
- if (fieldSpec == null)
- throw new ExplicitException("No field spec found: " + oneDevFieldName);
- labelMappings.put(mapping.getGiteaIssueLabel(), new Pair<>(fieldSpec, oneDevFieldValue));
- }
-
- for (Milestone milestone: oneDevProject.getMilestones())
- milestoneMappings.put(milestone.getName(), milestone);
-
- String initialIssueState = getIssueSetting().getInitialStateSpec().getName();
-
- List<Issue> issues = new ArrayList<>();
-
- Map<Long, Long> issueNumberMappings = new HashMap<>();
-
- AtomicInteger numOfImportedIssues = new AtomicInteger(0);
- PageDataConsumer pageDataConsumer = new PageDataConsumer() {
-
- private String joinAsMultilineHtml(List<String> values) {
- List<String> escapedValues = new ArrayList<>();
- for (String value: values)
- escapedValues.add(HtmlEscape.escapeHtml5(value));
- return StringUtils.join(escapedValues, "<br>");
- }
-
- @Override
- public void consume(List<JsonNode> pageData) throws InterruptedException {
- for (JsonNode issueNode: pageData) {
- if (Thread.interrupted())
- throw new InterruptedException();
-
- Map<String, String> extraIssueInfo = new LinkedHashMap<>();
-
- Issue issue = new Issue();
- issue.setProject(oneDevProject);
- issue.setTitle(issueNode.get("title").asText());
- issue.setDescription(issueNode.get("body").asText(null));
-
- issue.setNumberScope(oneDevProject.getForkRoot());
-
- Long oldNumber = issueNode.get("number").asLong();
- Long newNumber;
- if (dryRun || useExistingIssueNumbers)
- newNumber = oldNumber;
- else
- newNumber = OneDev.getInstance(IssueManager.class).getNextNumber(oneDevProject);
- issue.setNumber(newNumber);
- issueNumberMappings.put(oldNumber, newNumber);
-
- String issueFQN = giteaRepo + "#" + oldNumber;
-
- if (issueNode.get("state").asText().equals("closed"))
- issue.setState(importOption.getClosedIssueState());
- else
- issue.setState(initialIssueState);
-
- if (issueNode.hasNonNull("milestone")) {
- String milestoneName = issueNode.get("milestone").get("title").asText();
- Milestone milestone = milestoneMappings.get(milestoneName);
- if (milestone != null) {
- IssueSchedule schedule = new IssueSchedule();
- schedule.setIssue(issue);
- schedule.setMilestone(milestone);
- issue.getSchedules().add(schedule);
- } else {
- extraIssueInfo.put("Milestone", milestoneName);
- nonExistentMilestones.add(milestoneName);
- }
- }
-
- JsonNode userNode = issueNode.get("user");
- User user = getUser(users, userNode, logger);
- if (user != null) {
- issue.setSubmitter(user);
- } else {
- issue.setSubmitter(OneDev.getInstance(UserManager.class).getUnknown());
- nonExistentLogins.add(userNode.get("login").asText());
- }
-
- String created_at = issueNode.get("created_at").asText();
- issue.setSubmitDate(Date.from(Instant.from(
- DateTimeFormatter.ISO_DATE_TIME.parse(created_at))));
-
- LastUpdate lastUpdate = new LastUpdate();
- lastUpdate.setActivity("Opened");
- lastUpdate.setDate(issue.getSubmitDate());
- lastUpdate.setUser(issue.getSubmitter());
- issue.setLastUpdate(lastUpdate);
-
- List<JsonNode> assigneeNodes = new ArrayList<>();
- if (issueNode.hasNonNull("assignees")) {
- for (JsonNode assigneeNode: issueNode.get("assignees"))
- assigneeNodes.add(assigneeNode);
- } else if (issueNode.hasNonNull("assignee")) {
- assigneeNodes.add(issueNode.get("assignee"));
- }
-
- for (JsonNode assigneeNode: assigneeNodes) {
- IssueField assigneeField = new IssueField();
- assigneeField.setIssue(issue);
- assigneeField.setName(importOption.getAssigneesIssueField());
- assigneeField.setType(InputSpec.USER);
-
- user = getUser(users, assigneeNode, logger);
- if (user != null) {
- assigneeField.setValue(user.getName());
- issue.getFields().add(assigneeField);
- } else {
- nonExistentLogins.add(assigneeNode.get("login").asText());
- }
- }
-
- if (importOption.getDueDateIssueField() != null) {
- String dueDate = issueNode.get("due_date").asText(null);
- if (dueDate != null) {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(importOption.getDueDateIssueField());
- issueField.setType(InputSpec.DATE);
- issueField.setValue(DateUtils.formatDate(Date.from(
- Instant.from(DateTimeFormatter.ISO_DATE_TIME.parse(dueDate)))));
- issue.getFields().add(issueField);
- }
- }
-
- List<String> currentUnmappedLabels = new ArrayList<>();
- for (JsonNode labelNode: issueNode.get("labels")) {
- String labelName = labelNode.get("name").asText();
- Pair<FieldSpec, String> mapped = labelMappings.get(labelName);
- if (mapped != null) {
- IssueField labelField = new IssueField();
- labelField.setIssue(issue);
- labelField.setName(mapped.getFirst().getName());
- labelField.setType(InputSpec.ENUMERATION);
- labelField.setValue(mapped.getSecond());
- labelField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));
- issue.getFields().add(labelField);
- } else {
- currentUnmappedLabels.add(labelName);
- unmappedIssueLabels.add(HtmlEscape.escapeHtml5(labelName));
- }
- }
-
- if (!currentUnmappedLabels.isEmpty())
- extraIssueInfo.put("Labels", joinAsMultilineHtml(currentUnmappedLabels));
-
- String apiEndpoint = server.getApiEndpoint("/repos/" + giteaRepo
- + "/issues/" + oldNumber + "/comments");
- for (JsonNode commentNode: list(client, apiEndpoint, logger)) {
- String commentContent = commentNode.get("body").asText(null);
- if (commentContent != null) {
- IssueComment comment = new IssueComment();
- comment.setIssue(issue);
- comment.setContent(commentContent);
-
- created_at = commentNode.get("created_at").asText();
- comment.setDate(Date.from(Instant.from(
- DateTimeFormatter.ISO_DATE_TIME.parse(created_at))));
-
- userNode = commentNode.get("user");
- user = getUser(users, userNode, logger);
- if (user != null) {
- comment.setUser(user);
- } else {
- comment.setUser(OneDev.getInstance(UserManager.class).getUnknown());
- nonExistentLogins.add(userNode.get("username").asText());
- }
- issue.getComments().add(comment);
- }
- }
-
- issue.setCommentCount(issue.getComments().size());
-
- Set<String> fieldAndValues = new HashSet<>();
- for (IssueField field: issue.getFields()) {
- String fieldAndValue = field.getName() + "::" + field.getValue();
- if (!fieldAndValues.add(fieldAndValue)) {
- String errorMessage = String.format(
- "Duplicate issue field mapping (issue: %s, field: %s)",
- issueFQN, fieldAndValue);
- throw new ExplicitException(errorMessage);
- }
- }
-
- if (!extraIssueInfo.isEmpty()) {
- StringBuilder builder = new StringBuilder("|");
- for (String key: extraIssueInfo.keySet())
- builder.append(key).append("|");
- builder.append("\n|");
- extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));
- builder.append("\n|");
- for (String value: extraIssueInfo.values())
- builder.append(value).append("|");
-
- if (issue.getDescription() != null)
- issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());
- else
- issue.setDescription(builder.toString());
- }
- issues.add(issue);
- }
- logger.log("Imported " + numOfImportedIssues.addAndGet(pageData.size()) + " issues");
- }
-
- };
-
- String apiEndpoint = server.getApiEndpoint("/repos/" + giteaRepo + "/issues?state=all");
- list(client, apiEndpoint, pageDataConsumer, logger);
-
- if (!dryRun) {
- ReferenceMigrator migrator = new ReferenceMigrator(Issue.class, issueNumberMappings);
- Dao dao = OneDev.getInstance(Dao.class);
- for (Issue issue: issues) {
- if (issue.getDescription() != null)
- issue.setDescription(migrator.migratePrefixed(issue.getDescription(), "#"));
-
- OneDev.getInstance(IssueManager.class).save(issue);
- for (IssueSchedule schedule: issue.getSchedules())
- dao.persist(schedule);
- for (IssueField field: issue.getFields())
- dao.persist(field);
- for (IssueComment comment: issue.getComments()) {
- comment.setContent(migrator.migratePrefixed(comment.getContent(), "#"));
- dao.persist(comment);
- }
- }
- }
-
- ImportResult result = new ImportResult();
- result.nonExistentLogins.addAll(nonExistentLogins);
- result.nonExistentMilestones.addAll(nonExistentMilestones);
- result.unmappedIssueLabels.addAll(unmappedIssueLabels);
-
- if (numOfImportedIssues.get() != 0)
- result.issuesImported = true;
-
- return result;
- } finally {
- client.close();
- }
- }
-
- static GlobalIssueSetting getIssueSetting() {
- return OneDev.getInstance(SettingManager.class).getIssueSetting();
- }
-
- static List<JsonNode> list(Client client, String apiEndpoint, TaskLogger logger) {
- List<JsonNode> result = new ArrayList<>();
- list(client, apiEndpoint, new PageDataConsumer() {
-
- @Override
- public void consume(List<JsonNode> pageData) {
- result.addAll(pageData);
- }
-
- }, logger);
- return result;
- }
-
- static void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer,
- TaskLogger logger) {
- URI uri;
- try {
- uri = new URIBuilder(apiEndpoint)
- .addParameter("limit", String.valueOf(PER_PAGE)).build();
- } catch (URISyntaxException e) {
- throw new RuntimeException(e);
- }
-
- int page = 1;
- while (true) {
- try {
- URIBuilder builder = new URIBuilder(uri);
- builder.addParameter("page", String.valueOf(page));
- List<JsonNode> pageData = new ArrayList<>();
- for (JsonNode each: JerseyUtils.get(client, builder.build().toString(), logger))
- pageData.add(each);
- pageDataConsumer.consume(pageData);
- if (pageData.size() < PER_PAGE)
- break;
- page++;
- } catch (URISyntaxException|InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
-}
diff --git a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/IssueImportSource.java b/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/IssueImportSource.java
deleted file mode 100644
index f83bae735b..0000000000
--- a/server-plugin/server-plugin-import-gitea/src/main/java/io/onedev/server/plugin/imports/gitea/IssueImportSource.java
+++ /dev/null
@@ -1,70 +0,0 @@
-package io.onedev.server.plugin.imports.gitea;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import javax.ws.rs.client.Client;
-
-import org.hibernate.validator.constraints.NotEmpty;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.databind.JsonNode;
-
-import io.onedev.commons.utils.TaskLogger;
-import io.onedev.server.web.editable.annotation.ChoiceProvider;
-import io.onedev.server.web.editable.annotation.Editable;
-import io.onedev.server.web.util.WicketUtils;
-
-@Editable
-public class IssueImportSource implements Serializable {
-
- private static final long serialVersionUID = 1L;
-
- private static final Logger logger = LoggerFactory.getLogger(IssueImportSource.class);
-
- private String repository;
-
- @Editable(order=400, name="Gitea Repository", description="Choose Gitea repository to import issues from")
- @ChoiceProvider("getRepositoryChoices")
- @NotEmpty
- public String getRepository() {
- return repository;
- }
-
- public void setRepository(String repository) {
- this.repository = repository;
- }
-
- @SuppressWarnings("unused")
- private static List<String> getRepositoryChoices() {
- List<String> choices = new ArrayList<>();
-
- ImportServer server = WicketUtils.getPage().getMetaData(ImportServer.META_DATA_KEY);
-
- Client client = server.newClient();
- try {
- String apiEndpoint = server.getApiEndpoint("/user/repos");
- for (JsonNode repoNode: ImportUtils.list(client, apiEndpoint, new TaskLogger() {
-
- @Override
- public void log(String message, String sessionId) {
- logger.info(message);
- }
-
- })) {
- String repoName = repoNode.get("name").asText();
- String ownerName = repoNode.get("owner").get("login").asText();
- choices.add(ownerName + "/" + repoName);
- }
- } finally {
- client.close();
- }
-
- Collections.sort(choices);
- return choices;
- }
-
-}
diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImporter.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImporter.java
index 72737851ec..0b39273fe4 100644
--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImporter.java
+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubIssueImporter.java
@@ -1,10 +1,8 @@
package io.onedev.server.plugin.imports.github;
-import static io.onedev.server.plugin.imports.github.ImportUtils.NAME;
-import static io.onedev.server.plugin.imports.github.ImportUtils.buildImportOption;
-import static io.onedev.server.plugin.imports.github.ImportUtils.importIssues;
-
+import java.io.Serializable;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -14,35 +12,81 @@ import io.onedev.commons.utils.TaskLogger;
import io.onedev.server.imports.IssueImporter;
import io.onedev.server.model.Project;
import io.onedev.server.model.User;
-import io.onedev.server.web.util.WicketUtils;
+import io.onedev.server.web.util.ImportStep;
-public class GitHubIssueImporter extends IssueImporter<ImportServer, IssueImportSource, IssueImportOption> {
+public class GitHubIssueImporter implements IssueImporter {
private static final long serialVersionUID = 1L;
+ private final ImportStep<ImportServer> serverStep = new ImportStep<ImportServer>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Authenticate to GitHub";
+ }
+
+ @Override
+ protected ImportServer newSetting() {
+ return new ImportServer();
+ }
+
+ };
+
+ private final ImportStep<ImportRepository> repositoryStep = new ImportStep<ImportRepository>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Choose repository";
+ }
+
+ @Override
+ protected ImportRepository newSetting() {
+ ImportRepository repository = new ImportRepository();
+ repository.server = serverStep.getSetting();
+ return repository;
+ }
+
+ };
+
+ private final ImportStep<IssueImportOption> optionStep = new ImportStep<IssueImportOption>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Specify import option";
+ }
+
+ @Override
+ protected IssueImportOption newSetting() {
+ return serverStep.getSetting().buildIssueImportOption(
+ Lists.newArrayList(repositoryStep.getSetting().getRepository()));
+ }
+
+ };
+
@Override
public String getName() {
- return NAME;
+ return GitHubPluginModule.NAME;
}
@Override
- public String doImport(ImportServer where, IssueImportSource what, IssueImportOption how, Project project,
- boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {
- logger.log("Importing issues from repository " + what.getRepository() + "...");
+ public String doImport(Project project, boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {
+ logger.log("Importing issues from repository " + repositoryStep.getSetting().getRepository() + "...");
Map<String, Optional<User>> users = new HashMap<>();
- return importIssues(where, what.getRepository(), project, retainIssueNumbers, how, users, dryRun, logger)
- .toHtml("Issues imported successfully");
+
+ ImportResult result = serverStep.getSetting().importIssues(repositoryStep.getSetting().getRepository(),
+ project,retainIssueNumbers, optionStep.getSetting(), users, dryRun, logger);
+ return result.toHtml("Issues imported successfully");
}
@Override
- public IssueImportSource getWhat(ImportServer where, TaskLogger logger) {
- WicketUtils.getPage().setMetaData(ImportServer.META_DATA_KEY, where);
- return new IssueImportSource();
+ public List<ImportStep<? extends Serializable>> getSteps() {
+ return Lists.newArrayList(serverStep, repositoryStep, optionStep);
}
- @Override
- public IssueImportOption getHow(ImportServer where, IssueImportSource what, TaskLogger logger) {
- return buildImportOption(where, Lists.newArrayList(what.getRepository()), logger);
- }
-
}
\ No newline at end of file
diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubPluginModule.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubPluginModule.java
index 6f187c92eb..6cb02e5279 100644
--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubPluginModule.java
+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubPluginModule.java
@@ -1,6 +1,5 @@
package io.onedev.server.plugin.imports.github;
-import java.io.Serializable;
import java.util.Collection;
import com.google.common.collect.Lists;
@@ -17,6 +16,8 @@ import io.onedev.server.imports.ProjectImporterContribution;
*/
public class GitHubPluginModule extends AbstractPluginModule {
+ static final String NAME = "GitHub";
+
@Override
protected void configure() {
super.configure();
@@ -24,7 +25,7 @@ public class GitHubPluginModule extends AbstractPluginModule {
contribute(ProjectImporterContribution.class, new ProjectImporterContribution() {
@Override
- public Collection<ProjectImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> getImporters() {
+ public Collection<ProjectImporter> getImporters() {
return Lists.newArrayList(new GitHubProjectImporter());
}
@@ -38,7 +39,7 @@ public class GitHubPluginModule extends AbstractPluginModule {
contribute(IssueImporterContribution.class, new IssueImporterContribution() {
@Override
- public Collection<IssueImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> getImporters() {
+ public Collection<IssueImporter> getImporters() {
return Lists.newArrayList(new GitHubIssueImporter());
}
diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImporter.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImporter.java
index 93dd9abb5f..c24c563544 100644
--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImporter.java
+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubProjectImporter.java
@@ -1,147 +1,113 @@
package io.onedev.server.plugin.imports.github;
-import static io.onedev.server.plugin.imports.github.ImportUtils.NAME;
-import static io.onedev.server.plugin.imports.github.ImportUtils.get;
-import static io.onedev.server.plugin.imports.github.ImportUtils.importIssues;
-import static io.onedev.server.plugin.imports.github.ImportUtils.list;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
+import java.io.Serializable;
import java.util.List;
-import java.util.Map;
-import java.util.Optional;
import java.util.stream.Collectors;
-import javax.ws.rs.client.Client;
-
-import org.apache.http.client.utils.URIBuilder;
-import org.joda.time.format.ISODateTimeFormat;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
import io.onedev.commons.utils.TaskLogger;
-import io.onedev.server.OneDev;
-import io.onedev.server.entitymanager.MilestoneManager;
-import io.onedev.server.entitymanager.ProjectManager;
import io.onedev.server.imports.ProjectImporter;
-import io.onedev.server.model.Milestone;
-import io.onedev.server.model.Project;
-import io.onedev.server.model.User;
-import io.onedev.server.storage.StorageManager;
+import io.onedev.server.web.util.ImportStep;
-public class GitHubProjectImporter extends ProjectImporter<ImportServer, ProjectImportSource, ProjectImportOption> {
+public class GitHubProjectImporter implements ProjectImporter {
private static final long serialVersionUID = 1L;
- @Override
- public String getName() {
- return NAME;
- }
+ private final ImportStep<ImportServer> serverStep = new ImportStep<ImportServer>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Authenticate to GitHub";
+ }
+
+ @Override
+ protected ImportServer newSetting() {
+ return new ImportServer();
+ }
+
+ };
- @Override
- public String doImport(ImportServer where, ProjectImportSource what, ProjectImportOption how,
- boolean dryRun, TaskLogger logger) {
- Collection<Long> projectIds = new ArrayList<>();
- Client client = where.newClient();
- try {
- Map<String, Optional<User>> users = new HashMap<>();
- ImportResult result = new ImportResult();
- for (ProjectMapping projectMapping: what.getProjectMappings()) {
- logger.log("Cloning code from repository " + projectMapping.getGitHubRepo() + "...");
-
- String apiEndpoint = where.getApiEndpoint("/repos/" + projectMapping.getGitHubRepo());
- JsonNode repoNode = get(client, apiEndpoint, logger);
- ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
- Project project = projectManager.initialize(projectMapping.getOneDevProject());
- Preconditions.checkState(project.isNew());
- project.setDescription(repoNode.get("description").asText(null));
- project.setIssueManagement(repoNode.get("has_issues").asBoolean());
-
- boolean isPrivate = repoNode.get("private").asBoolean();
- if (!isPrivate && how.getPublicRole() != null)
- project.setDefaultRole(how.getPublicRole());
-
- URIBuilder builder = new URIBuilder(repoNode.get("clone_url").asText());
- if (isPrivate)
- builder.setUserInfo("git", where.getAccessToken());
-
- if (!dryRun) {
- projectManager.clone(project, builder.build().toString());
- projectIds.add(project.getId());
- }
-
- if (how.getIssueImportOption() != null) {
- List<Milestone> milestones = new ArrayList<>();
- logger.log("Importing milestones from repository " + projectMapping.getGitHubRepo() + "...");
- apiEndpoint = where.getApiEndpoint("/repos/" + projectMapping.getGitHubRepo() + "/milestones?state=all");
- for (JsonNode milestoneNode: list(client, apiEndpoint, logger)) {
- Milestone milestone = new Milestone();
- milestone.setName(milestoneNode.get("title").asText());
- milestone.setDescription(milestoneNode.get("description").asText(null));
- milestone.setProject(project);
- String dueDateString = milestoneNode.get("due_on").asText(null);
- if (dueDateString != null)
- milestone.setDueDate(ISODateTimeFormat.dateTimeNoMillis().parseDateTime(dueDateString).toDate());
- if (milestoneNode.get("state").asText().equals("closed"))
- milestone.setClosed(true);
-
- milestones.add(milestone);
- project.getMilestones().add(milestone);
-
- if (!dryRun)
- OneDev.getInstance(MilestoneManager.class).save(milestone);
- }
-
- logger.log("Importing issues from repository " + projectMapping.getGitHubRepo() + "...");
- ImportResult currentResult = importIssues(where, projectMapping.getGitHubRepo(),
- project, true, how.getIssueImportOption(), users, dryRun, logger);
- result.nonExistentLogins.addAll(currentResult.nonExistentLogins);
- result.nonExistentMilestones.addAll(currentResult.nonExistentMilestones);
- result.unmappedIssueLabels.addAll(currentResult.unmappedIssueLabels);
-
- result.issuesImported |= currentResult.issuesImported;
- }
- }
-
- return result.toHtml("Repositories imported successfully");
- } catch (Exception e) {
- for (Long projectId: projectIds)
- OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);
- throw new RuntimeException(e);
- } finally {
- client.close();
+ private final ImportStep<ImportOrganization> organizationStep = new ImportStep<ImportOrganization>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Choose organization";
}
- }
- @Override
- public ProjectImportSource getWhat(ImportServer where, TaskLogger logger) {
- ProjectImportSource importSource = new ProjectImportSource();
- Client client = where.newClient();
- try {
- String apiEndpoint = where.getApiEndpoint("/user/repos");
- for (JsonNode repoNode: list(client, apiEndpoint, logger)) {
- String repoName = repoNode.get("name").asText();
- String ownerName = repoNode.get("owner").get("login").asText();
+ @Override
+ protected ImportOrganization newSetting() {
+ ImportOrganization organization = new ImportOrganization();
+ organization.server = serverStep.getSetting();
+ return organization;
+ }
+
+ };
+
+ private final ImportStep<ImportRepositories> repositoriesStep = new ImportStep<ImportRepositories>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Specify repositories";
+ }
+
+ @Override
+ protected ImportRepositories newSetting() {
+ ImportRepositories repositories = new ImportRepositories();
+ String organization = organizationStep.getSetting().getOrganization();
+ for (String repository: serverStep.getSetting().listRepositories(organization)) {
ProjectMapping projectMapping = new ProjectMapping();
- projectMapping.setGitHubRepo(ownerName + "/" + repoName);
- projectMapping.setOneDevProject(ownerName + "/" + repoName);
- importSource.getProjectMappings().add(projectMapping);
- }
- } finally {
- client.close();
+ projectMapping.setGitHubRepo(repository);
+ projectMapping.setOneDevProject(repository);
+ repositories.getProjectMappings().add(projectMapping);
+ }
+ return repositories;
+ }
+
+ };
+
+ private final ImportStep<ProjectImportOption> optionStep = new ImportStep<ProjectImportOption>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Specify import option";
}
- return importSource;
+
+ @Override
+ protected ProjectImportOption newSetting() {
+ ProjectImportOption option = new ProjectImportOption();
+ List<String> gitHubRepos = repositoriesStep.getSetting().getProjectMappings().stream()
+ .map(it->it.getGitHubRepo()).collect(Collectors.toList());
+ option.setIssueImportOption(serverStep.getSetting().buildIssueImportOption(gitHubRepos));
+ return option;
+ }
+
+ };
+
+ @Override
+ public String getName() {
+ return GitHubPluginModule.NAME;
+ }
+
+ @Override
+ public String doImport(boolean dryRun, TaskLogger logger) {
+ ImportServer server = serverStep.getSetting();
+ ImportRepositories repositories = repositoriesStep.getSetting();
+ ProjectImportOption option = optionStep.getSetting();
+ return server.importProjects(repositories, option, dryRun, logger);
}
@Override
- public ProjectImportOption getHow(ImportServer where, ProjectImportSource what, TaskLogger logger) {
- ProjectImportOption importOption = new ProjectImportOption();
- List<String> gitHubRepos = what.getProjectMappings().stream()
- .map(it->it.getGitHubRepo()).collect(Collectors.toList());
- importOption.setIssueImportOption(ImportUtils.buildImportOption(where, gitHubRepos, logger));
- return importOption;
+ public List<ImportStep<? extends Serializable>> getSteps() {
+ return Lists.newArrayList(serverStep, organizationStep, repositoriesStep, optionStep);
}
}
\ No newline at end of file
diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportOrganization.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportOrganization.java
new file mode 100644
index 0000000000..70b4d179e2
--- /dev/null
+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportOrganization.java
@@ -0,0 +1,38 @@
+package io.onedev.server.plugin.imports.github;
+
+import java.io.Serializable;
+import java.util.List;
+
+import io.onedev.server.util.ComponentContext;
+import io.onedev.server.web.editable.BeanEditor;
+import io.onedev.server.web.editable.annotation.ChoiceProvider;
+import io.onedev.server.web.editable.annotation.Editable;
+
+@Editable
+public class ImportOrganization implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ ImportServer server;
+
+ private String organization;
+
+ @Editable(order=100, name="GitHub Organization", description="Select organization to import from. "
+ + "Leave empty to import from repositories under current account")
+ @ChoiceProvider("getOrganizationChoices")
+ public String getOrganization() {
+ return organization;
+ }
+
+ public void setOrganization(String organization) {
+ this.organization = organization;
+ }
+
+ @SuppressWarnings("unused")
+ private static List<String> getOrganizationChoices() {
+ BeanEditor editor = ComponentContext.get().getComponent().findParent(BeanEditor.class);
+ ImportOrganization setting = (ImportOrganization) editor.getModelObject();
+ return setting.server.listOrganizations();
+ }
+
+}
diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ProjectImportSource.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportRepositories.java
similarity index 93%
rename from server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ProjectImportSource.java
rename to server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportRepositories.java
index f6c986334b..71c75f5ad9 100644
--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ProjectImportSource.java
+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportRepositories.java
@@ -15,17 +15,19 @@ import io.onedev.server.model.Project;
import io.onedev.server.util.validation.Validatable;
import io.onedev.server.util.validation.annotation.ClassValidating;
import io.onedev.server.web.editable.annotation.Editable;
+import io.onedev.server.web.editable.annotation.OmitName;
@Editable
@ClassValidating
-public class ProjectImportSource implements Serializable, Validatable {
+public class ImportRepositories implements Serializable, Validatable {
private static final long serialVersionUID = 1L;
private List<ProjectMapping> projectMappings = new ArrayList<>();
- @Editable(order=100, name="Repositories to Import")
+ @Editable(order=100)
@Size(min=1, max=10000, message="No repositories to import")
+ @OmitName
public List<ProjectMapping> getProjectMappings() {
return projectMappings;
}
diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportRepository.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportRepository.java
new file mode 100644
index 0000000000..7bf699b352
--- /dev/null
+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportRepository.java
@@ -0,0 +1,39 @@
+package io.onedev.server.plugin.imports.github;
+
+import java.util.List;
+
+import org.hibernate.validator.constraints.NotEmpty;
+
+import io.onedev.server.util.ComponentContext;
+import io.onedev.server.util.EditContext;
+import io.onedev.server.web.editable.BeanEditor;
+import io.onedev.server.web.editable.annotation.ChoiceProvider;
+import io.onedev.server.web.editable.annotation.Editable;
+
+@Editable
+public class ImportRepository extends ImportOrganization {
+
+ private static final long serialVersionUID = 1L;
+
+ private String repository;
+
+ @Editable(order=200, name="GitHub Repository", description="Select repository to import from")
+ @ChoiceProvider("getRepositoryChoices")
+ @NotEmpty
+ public String getRepository() {
+ return repository;
+ }
+
+ public void setRepository(String repository) {
+ this.repository = repository;
+ }
+
+ @SuppressWarnings("unused")
+ private static List<String> getRepositoryChoices() {
+ BeanEditor editor = ComponentContext.get().getComponent().findParent(BeanEditor.class);
+ ImportRepository setting = (ImportRepository) editor.getModelObject();
+ String organization = (String) EditContext.get().getInputValue("organization");
+ return setting.server.listRepositories(organization);
+ }
+
+}
diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportServer.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportServer.java
index 9b3811bb13..1366d6b75f 100644
--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportServer.java
+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportServer.java
@@ -1,7 +1,21 @@
package io.onedev.server.plugin.imports.github;
import java.io.Serializable;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.annotation.Nullable;
import javax.validation.ConstraintValidatorContext;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
@@ -9,13 +23,45 @@ import javax.ws.rs.client.Invocation;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response;
-import org.apache.wicket.MetaDataKey;
+import org.apache.http.client.utils.URIBuilder;
import org.glassfish.jersey.client.ClientProperties;
import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
import org.hibernate.validator.constraints.NotEmpty;
+import org.joda.time.format.ISODateTimeFormat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.unbescape.html.HtmlEscape;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.base.Preconditions;
+
+import edu.emory.mathcs.backport.java.util.Collections;
+import io.onedev.commons.utils.ExplicitException;
import io.onedev.commons.utils.StringUtils;
+import io.onedev.commons.utils.TaskLogger;
+import io.onedev.server.OneDev;
+import io.onedev.server.entitymanager.IssueManager;
+import io.onedev.server.entitymanager.MilestoneManager;
+import io.onedev.server.entitymanager.ProjectManager;
+import io.onedev.server.entitymanager.SettingManager;
+import io.onedev.server.entitymanager.UserManager;
+import io.onedev.server.entityreference.ReferenceMigrator;
+import io.onedev.server.model.Issue;
+import io.onedev.server.model.IssueComment;
+import io.onedev.server.model.IssueField;
+import io.onedev.server.model.IssueSchedule;
+import io.onedev.server.model.Milestone;
+import io.onedev.server.model.Project;
+import io.onedev.server.model.User;
+import io.onedev.server.model.support.LastUpdate;
+import io.onedev.server.model.support.administration.GlobalIssueSetting;
+import io.onedev.server.model.support.inputspec.InputSpec;
+import io.onedev.server.model.support.issue.field.spec.FieldSpec;
+import io.onedev.server.persistence.dao.Dao;
+import io.onedev.server.storage.StorageManager;
import io.onedev.server.util.JerseyUtils;
+import io.onedev.server.util.JerseyUtils.PageDataConsumer;
+import io.onedev.server.util.Pair;
import io.onedev.server.util.validation.Validatable;
import io.onedev.server.util.validation.annotation.ClassValidating;
import io.onedev.server.web.editable.annotation.Editable;
@@ -27,20 +73,18 @@ public class ImportServer implements Serializable, Validatable {
private static final long serialVersionUID = 1L;
- static final MetaDataKey<ImportServer> META_DATA_KEY = new MetaDataKey<ImportServer>() {
-
- private static final long serialVersionUID = 1L;
-
- };
+ private static final Logger logger = LoggerFactory.getLogger(ImportServer.class);
+
+ private static final int PER_PAGE = 50;
- static final String PROP_API_URL = "apiUrl";
+ private static final String PROP_API_URL = "apiUrl";
- static final String PROP_ACCESS_TOKEN = "accessToken";
+ private static final String PROP_ACCESS_TOKEN = "accessToken";
private String apiUrl = "https://api.github.com";
private String accessToken;
-
+
@Editable(order=10, name="GitHub API URL", description="Specify GitHub API url, for instance <tt>https://api.github.com</tt>")
@NotEmpty
public String getApiUrl() {
@@ -63,10 +107,502 @@ public class ImportServer implements Serializable, Validatable {
this.accessToken = accessToken;
}
- public String getApiEndpoint(String apiPath) {
+ private String getApiEndpoint(String apiPath) {
return StringUtils.stripEnd(apiUrl, "/") + "/" + StringUtils.stripStart(apiPath, "/");
}
+
+ List<String> listOrganizations() {
+ List<String> organizations = new ArrayList<>();
+
+ Client client = newClient();
+ try {
+ String apiEndpoint = getApiEndpoint("/user/orgs");
+ for (JsonNode orgNode: list(client, apiEndpoint, new TaskLogger() {
+
+ @Override
+ public void log(String message, String sessionId) {
+ logger.info(message);
+ }
+
+ })) {
+ organizations.add(orgNode.get("login").asText());
+ }
+ Collections.sort(organizations);
+ } catch (Exception e) {
+ logger.error("Error listing organizations", e);
+ } finally {
+ client.close();
+ }
+
+ return organizations;
+ }
+
+ List<String> listRepositories(@Nullable String organization) {
+ Client client = newClient();
+ try {
+ String apiEndpoint;
+ if (organization != null)
+ apiEndpoint = getApiEndpoint("/orgs/" + organization + "/repos");
+ else
+ apiEndpoint = getApiEndpoint("/user/repos?type=owner");
+ List<String> repositories = new ArrayList<>();
+ for (JsonNode repoNode: list(client, apiEndpoint, new TaskLogger() {
+
+ @Override
+ public void log(String message, String sessionId) {
+ logger.info(message);
+ }
+
+ })) {
+ String repoName = repoNode.get("name").asText();
+ String ownerName = repoNode.get("owner").get("login").asText();
+ repositories.add(ownerName + "/" + repoName);
+ }
+ Collections.sort(repositories);
+ return repositories;
+ } finally {
+ client.close();
+ }
+
+ }
+
+ private Client newClient() {
+ Client client = ClientBuilder.newClient();
+ client.property(ClientProperties.FOLLOW_REDIRECTS, true);
+ client.register(HttpAuthenticationFeature.basic("git", getAccessToken()));
+ return client;
+ }
+
+ IssueImportOption buildIssueImportOption(Collection<String> gitHubRepos) {
+ IssueImportOption importOption = new IssueImportOption();
+ Client client = newClient();
+ try {
+ TaskLogger taskLogger = new TaskLogger() {
+
+ @Override
+ public void log(String message, String sessionId) {
+ logger.info(message);
+ }
+
+ };
+ Set<String> labels = new LinkedHashSet<>();
+ for (String each: gitHubRepos) {
+ String apiEndpoint = getApiEndpoint("/repos/" + each + "/labels");
+ for (JsonNode labelNode: list(client, apiEndpoint, taskLogger))
+ labels.add(labelNode.get("name").asText());
+ }
+
+ for (String label: labels) {
+ IssueLabelMapping mapping = new IssueLabelMapping();
+ mapping.setGitHubIssueLabel(label);
+ importOption.getIssueLabelMappings().add(mapping);
+ }
+ } finally {
+ client.close();
+ }
+ return importOption;
+ }
+
+ @Nullable
+ private User getUser(Client client, Map<String, Optional<User>> users,
+ String login, TaskLogger logger) {
+ Optional<User> userOpt = users.get(login);
+ if (userOpt == null) {
+ String apiEndpoint = getApiEndpoint("/users/" + login);
+ String email = get(client, apiEndpoint, logger).get("email").asText(null);
+ if (email != null)
+ userOpt = Optional.ofNullable(OneDev.getInstance(UserManager.class).findByVerifiedEmailAddress(email));
+ else
+ userOpt = Optional.empty();
+ users.put(login, userOpt);
+ }
+ return userOpt.orElse(null);
+ }
+
+ ImportResult importIssues(String gitHubRepo, Project oneDevProject, boolean useExistingIssueNumbers,
+ IssueImportOption importOption, Map<String, Optional<User>> users, boolean dryRun, TaskLogger logger) {
+ Client client = newClient();
+ try {
+ Set<String> nonExistentMilestones = new HashSet<>();
+ Set<String> nonExistentLogins = new HashSet<>();
+ Set<String> unmappedIssueLabels = new HashSet<>();
+
+ Map<String, Pair<FieldSpec, String>> labelMappings = new HashMap<>();
+ Map<String, Milestone> milestoneMappings = new HashMap<>();
+
+ for (IssueLabelMapping mapping: importOption.getIssueLabelMappings()) {
+ String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");
+ String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");
+ FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);
+ if (fieldSpec == null)
+ throw new ExplicitException("No field spec found: " + oneDevFieldName);
+ labelMappings.put(mapping.getGitHubIssueLabel(), new Pair<>(fieldSpec, oneDevFieldValue));
+ }
+
+ for (Milestone milestone: oneDevProject.getMilestones())
+ milestoneMappings.put(milestone.getName(), milestone);
+
+ String initialIssueState = getIssueSetting().getInitialStateSpec().getName();
+
+ List<Issue> issues = new ArrayList<>();
+
+ Map<Long, Long> issueNumberMappings = new HashMap<>();
+
+ AtomicInteger numOfImportedIssues = new AtomicInteger(0);
+ PageDataConsumer pageDataConsumer = new PageDataConsumer() {
+
+ private String joinAsMultilineHtml(List<String> values) {
+ List<String> escapedValues = new ArrayList<>();
+ for (String value: values)
+ escapedValues.add(HtmlEscape.escapeHtml5(value));
+ return StringUtils.join(escapedValues, "<br>");
+ }
+
+ @Override
+ public void consume(List<JsonNode> pageData) throws InterruptedException {
+ for (JsonNode issueNode: pageData) {
+ if (Thread.interrupted())
+ throw new InterruptedException();
+
+ if (issueNode.get("pull_request") != null)
+ continue;
+
+ Map<String, String> extraIssueInfo = new LinkedHashMap<>();
+
+ Issue issue = new Issue();
+ issue.setProject(oneDevProject);
+ issue.setTitle(issueNode.get("title").asText());
+ issue.setDescription(issueNode.get("body").asText(null));
+ issue.setNumberScope(oneDevProject.getForkRoot());
+ Long oldNumber = issueNode.get("number").asLong();
+ Long newNumber;
+ if (dryRun || useExistingIssueNumbers)
+ newNumber = oldNumber;
+ else
+ newNumber = OneDev.getInstance(IssueManager.class).getNextNumber(oneDevProject);
+ issue.setNumber(newNumber);
+ issueNumberMappings.put(oldNumber, newNumber);
+
+ if (issueNode.get("state").asText().equals("closed"))
+ issue.setState(importOption.getClosedIssueState());
+ else
+ issue.setState(initialIssueState);
+
+ if (issueNode.hasNonNull("milestone")) {
+ String milestoneName = issueNode.get("milestone").get("title").asText();
+ Milestone milestone = milestoneMappings.get(milestoneName);
+ if (milestone != null) {
+ IssueSchedule schedule = new IssueSchedule();
+ schedule.setIssue(issue);
+ schedule.setMilestone(milestone);
+ issue.getSchedules().add(schedule);
+ } else {
+ extraIssueInfo.put("Milestone", milestoneName);
+ nonExistentMilestones.add(milestoneName);
+ }
+ }
+
+ String login = issueNode.get("user").get("login").asText(null);
+ User user = getUser(client, users, login, logger);
+ if (user != null) {
+ issue.setSubmitter(user);
+ } else {
+ issue.setSubmitter(OneDev.getInstance(UserManager.class).getUnknown());
+ nonExistentLogins.add(login);
+ }
+
+ issue.setSubmitDate(ISODateTimeFormat.dateTimeNoMillis()
+ .parseDateTime(issueNode.get("created_at").asText())
+ .toDate());
+
+ LastUpdate lastUpdate = new LastUpdate();
+ lastUpdate.setActivity("Opened");
+ lastUpdate.setDate(issue.getSubmitDate());
+ lastUpdate.setUser(issue.getSubmitter());
+ issue.setLastUpdate(lastUpdate);
+
+ for (JsonNode assigneeNode: issueNode.get("assignees")) {
+ IssueField assigneeField = new IssueField();
+ assigneeField.setIssue(issue);
+ assigneeField.setName(importOption.getAssigneesIssueField());
+ assigneeField.setType(InputSpec.USER);
+
+ login = assigneeNode.get("login").asText();
+ user = getUser(client, users, login, logger);
+ if (user != null) {
+ assigneeField.setValue(user.getName());
+ issue.getFields().add(assigneeField);
+ } else {
+ nonExistentLogins.add(login);
+ }
+ }
+
+ String apiEndpoint = getApiEndpoint("/repos/" + gitHubRepo
+ + "/issues/" + oldNumber + "/comments");
+ for (JsonNode commentNode: list(client, apiEndpoint, logger)) {
+ IssueComment comment = new IssueComment();
+ comment.setIssue(issue);
+ comment.setContent(commentNode.get("body").asText(null));
+ comment.setDate(ISODateTimeFormat.dateTimeNoMillis()
+ .parseDateTime(commentNode.get("created_at").asText())
+ .toDate());
+
+ login = commentNode.get("user").get("login").asText();
+ user = getUser(client, users, login, logger);
+ if (user != null) {
+ comment.setUser(user);
+ } else {
+ comment.setUser(OneDev.getInstance(UserManager.class).getUnknown());
+ nonExistentLogins.add(login);
+ }
+
+ issue.getComments().add(comment);
+ }
+
+ issue.setCommentCount(issue.getComments().size());
+
+ apiEndpoint = getApiEndpoint("/repos/" + gitHubRepo + "/issues/" + oldNumber + "/labels");
+ List<String> currentUnmappedLabels = new ArrayList<>();
+ for (JsonNode labelNode: list(client, apiEndpoint, logger)) {
+ String labelName = labelNode.get("name").asText();
+ Pair<FieldSpec, String> mapped = labelMappings.get(labelName);
+ if (mapped != null) {
+ IssueField tagField = new IssueField();
+ tagField.setIssue(issue);
+ tagField.setName(mapped.getFirst().getName());
+ tagField.setType(InputSpec.ENUMERATION);
+ tagField.setValue(mapped.getSecond());
+ tagField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));
+ issue.getFields().add(tagField);
+ } else {
+ currentUnmappedLabels.add(labelName);
+ unmappedIssueLabels.add(HtmlEscape.escapeHtml5(labelName));
+ }
+ }
+
+ if (!currentUnmappedLabels.isEmpty())
+ extraIssueInfo.put("Labels", joinAsMultilineHtml(currentUnmappedLabels));
+
+ Set<String> fieldAndValues = new HashSet<>();
+ for (IssueField field: issue.getFields()) {
+ String fieldAndValue = field.getName() + "::" + field.getValue();
+ if (!fieldAndValues.add(fieldAndValue)) {
+ String errorMessage = String.format(
+ "Duplicate issue field mapping (issue: %s, field: %s)",
+ gitHubRepo + "#" + oldNumber, fieldAndValue);
+ throw new ExplicitException(errorMessage);
+ }
+ }
+
+ if (!extraIssueInfo.isEmpty()) {
+ StringBuilder builder = new StringBuilder("|");
+ for (String key: extraIssueInfo.keySet())
+ builder.append(key).append("|");
+ builder.append("\n|");
+ extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));
+ builder.append("\n|");
+ for (String value: extraIssueInfo.values())
+ builder.append(value).append("|");
+
+ if (issue.getDescription() != null)
+ issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());
+ else
+ issue.setDescription(builder.toString());
+ }
+ issues.add(issue);
+ numOfImportedIssues.incrementAndGet();
+ }
+ logger.log("Imported " + numOfImportedIssues.get() + " issues");
+ }
+
+ };
+
+ String apiEndpoint = getApiEndpoint("/repos/" + gitHubRepo + "/issues?state=all&direction=asc");
+ list(client, apiEndpoint, pageDataConsumer, logger);
+
+ if (!dryRun) {
+ ReferenceMigrator migrator = new ReferenceMigrator(Issue.class, issueNumberMappings);
+ Dao dao = OneDev.getInstance(Dao.class);
+ for (Issue issue: issues) {
+ if (issue.getDescription() != null)
+ issue.setDescription(migrator.migratePrefixed(issue.getDescription(), "#"));
+
+ OneDev.getInstance(IssueManager.class).save(issue);
+ for (IssueSchedule schedule: issue.getSchedules())
+ dao.persist(schedule);
+ for (IssueField field: issue.getFields())
+ dao.persist(field);
+ for (IssueComment comment: issue.getComments()) {
+ comment.setContent(migrator.migratePrefixed(comment.getContent(), "#"));
+ dao.persist(comment);
+ }
+ }
+ }
+
+ ImportResult result = new ImportResult();
+ result.nonExistentLogins.addAll(nonExistentLogins);
+ result.nonExistentMilestones.addAll(nonExistentMilestones);
+ result.unmappedIssueLabels.addAll(unmappedIssueLabels);
+
+ if (numOfImportedIssues.get() != 0)
+ result.issuesImported = true;
+
+ return result;
+ } finally {
+ client.close();
+ }
+ }
+
+ private GlobalIssueSetting getIssueSetting() {
+ return OneDev.getInstance(SettingManager.class).getIssueSetting();
+ }
+
+ private List<JsonNode> list(Client client, String apiEndpoint, TaskLogger logger) {
+ List<JsonNode> result = new ArrayList<>();
+ list(client, apiEndpoint, new PageDataConsumer() {
+
+ @Override
+ public void consume(List<JsonNode> pageData) {
+ result.addAll(pageData);
+ }
+
+ }, logger);
+ return result;
+ }
+
+ private void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer,
+ TaskLogger logger) {
+ URI uri;
+ try {
+ uri = new URIBuilder(apiEndpoint)
+ .addParameter("per_page", String.valueOf(PER_PAGE)).build();
+ } catch (URISyntaxException e) {
+ throw new RuntimeException(e);
+ }
+
+ int page = 1;
+ while (true) {
+ try {
+ URIBuilder builder = new URIBuilder(uri);
+ builder.addParameter("page", String.valueOf(page));
+ List<JsonNode> pageData = new ArrayList<>();
+ for (JsonNode each: get(client, builder.build().toString(), logger))
+ pageData.add(each);
+ pageDataConsumer.consume(pageData);
+ if (pageData.size() < PER_PAGE)
+ break;
+ page++;
+ } catch (URISyntaxException|InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private JsonNode get(Client client, String apiEndpoint, TaskLogger logger) {
+ WebTarget target = client.target(apiEndpoint);
+ Invocation.Builder builder = target.request();
+ while (true) {
+ try (Response response = builder.get()) {
+ int status = response.getStatus();
+ if (status != 200) {
+ String errorMessage = response.readEntity(String.class);
+ if (StringUtils.isNotBlank(errorMessage)) {
+ if (errorMessage.contains("rate limit exceeded")) {
+ long resetTime = Long.parseLong(response.getHeaderString("x-ratelimit-reset"))*1000L;
+ logger.log("Rate limit exceeded, wait until reset...");
+ try {
+ Thread.sleep(resetTime + 60*1000L - System.currentTimeMillis());
+ continue;
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ throw new ExplicitException(String.format("Http request failed (url: %s, status code: %d, error message: %s)",
+ apiEndpoint, status, errorMessage));
+ }
+ } else {
+ throw new ExplicitException(String.format("Http request failed (status: %s)", status));
+ }
+ }
+ return response.readEntity(JsonNode.class);
+ }
+ }
+ }
+
+ String importProjects(ImportRepositories repositories, ProjectImportOption option, boolean dryRun, TaskLogger logger) {
+ Collection<Long> projectIds = new ArrayList<>();
+ Client client = newClient();
+ try {
+ Map<String, Optional<User>> users = new HashMap<>();
+ ImportResult result = new ImportResult();
+ for (ProjectMapping projectMapping: repositories.getProjectMappings()) {
+ logger.log("Cloning code from repository " + projectMapping.getGitHubRepo() + "...");
+
+ String apiEndpoint = getApiEndpoint("/repos/" + projectMapping.getGitHubRepo());
+ JsonNode repoNode = get(client, apiEndpoint, logger);
+ ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
+ Project project = projectManager.initialize(projectMapping.getOneDevProject());
+ Preconditions.checkState(project.isNew());
+ project.setDescription(repoNode.get("description").asText(null));
+ project.setIssueManagement(repoNode.get("has_issues").asBoolean());
+
+ boolean isPrivate = repoNode.get("private").asBoolean();
+ if (!isPrivate && option.getPublicRole() != null)
+ project.setDefaultRole(option.getPublicRole());
+
+ URIBuilder builder = new URIBuilder(repoNode.get("clone_url").asText());
+ if (isPrivate)
+ builder.setUserInfo("git", getAccessToken());
+
+ if (!dryRun) {
+ projectManager.clone(project, builder.build().toString());
+ projectIds.add(project.getId());
+ }
+
+ if (option.getIssueImportOption() != null) {
+ List<Milestone> milestones = new ArrayList<>();
+ logger.log("Importing milestones from repository " + projectMapping.getGitHubRepo() + "...");
+ apiEndpoint = getApiEndpoint("/repos/" + projectMapping.getGitHubRepo() + "/milestones?state=all");
+ for (JsonNode milestoneNode: list(client, apiEndpoint, logger)) {
+ Milestone milestone = new Milestone();
+ milestone.setName(milestoneNode.get("title").asText());
+ milestone.setDescription(milestoneNode.get("description").asText(null));
+ milestone.setProject(project);
+ String dueDateString = milestoneNode.get("due_on").asText(null);
+ if (dueDateString != null)
+ milestone.setDueDate(ISODateTimeFormat.dateTimeNoMillis().parseDateTime(dueDateString).toDate());
+ if (milestoneNode.get("state").asText().equals("closed"))
+ milestone.setClosed(true);
+
+ milestones.add(milestone);
+ project.getMilestones().add(milestone);
+
+ if (!dryRun)
+ OneDev.getInstance(MilestoneManager.class).save(milestone);
+ }
+
+ logger.log("Importing issues from repository " + projectMapping.getGitHubRepo() + "...");
+ ImportResult currentResult = importIssues(projectMapping.getGitHubRepo(),
+ project, true, option.getIssueImportOption(), users, dryRun, logger);
+ result.nonExistentLogins.addAll(currentResult.nonExistentLogins);
+ result.nonExistentMilestones.addAll(currentResult.nonExistentMilestones);
+ result.unmappedIssueLabels.addAll(currentResult.unmappedIssueLabels);
+
+ result.issuesImported |= currentResult.issuesImported;
+ }
+ }
+
+ return result.toHtml("Repositories imported successfully");
+ } catch (Exception e) {
+ for (Long projectId: projectIds)
+ OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);
+ throw new RuntimeException(e);
+ } finally {
+ client.close();
+ }
+ }
+
@Override
public boolean isValid(ConstraintValidatorContext context) {
Client client = ClientBuilder.newClient();
@@ -112,11 +648,4 @@ public class ImportServer implements Serializable, Validatable {
return true;
}
- Client newClient() {
- Client client = ClientBuilder.newClient();
- client.property(ClientProperties.FOLLOW_REDIRECTS, true);
- client.register(HttpAuthenticationFeature.basic("git", getAccessToken()));
- return client;
- }
-
}
diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportUtils.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportUtils.java
deleted file mode 100644
index 4c7c6d8276..0000000000
--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/ImportUtils.java
+++ /dev/null
@@ -1,405 +0,0 @@
-package io.onedev.server.plugin.imports.github;
-
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import javax.annotation.Nullable;
-import javax.ws.rs.client.Client;
-import javax.ws.rs.client.Invocation;
-import javax.ws.rs.client.WebTarget;
-import javax.ws.rs.core.Response;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.http.client.utils.URIBuilder;
-import org.joda.time.format.ISODateTimeFormat;
-import org.unbescape.html.HtmlEscape;
-
-import com.fasterxml.jackson.databind.JsonNode;
-
-import io.onedev.commons.utils.ExplicitException;
-import io.onedev.commons.utils.TaskLogger;
-import io.onedev.server.OneDev;
-import io.onedev.server.entitymanager.IssueManager;
-import io.onedev.server.entitymanager.SettingManager;
-import io.onedev.server.entitymanager.UserManager;
-import io.onedev.server.entityreference.ReferenceMigrator;
-import io.onedev.server.model.Issue;
-import io.onedev.server.model.IssueComment;
-import io.onedev.server.model.IssueField;
-import io.onedev.server.model.IssueSchedule;
-import io.onedev.server.model.Milestone;
-import io.onedev.server.model.Project;
-import io.onedev.server.model.User;
-import io.onedev.server.model.support.LastUpdate;
-import io.onedev.server.model.support.administration.GlobalIssueSetting;
-import io.onedev.server.model.support.inputspec.InputSpec;
-import io.onedev.server.model.support.issue.field.spec.FieldSpec;
-import io.onedev.server.persistence.dao.Dao;
-import io.onedev.server.util.JerseyUtils.PageDataConsumer;
-import io.onedev.server.util.Pair;
-
-public class ImportUtils {
-
- static final String NAME = "GitHub";
-
- static final int PER_PAGE = 50;
-
- static IssueImportOption buildImportOption(ImportServer server, Collection<String> gitHubRepos, TaskLogger logger) {
- IssueImportOption importOption = new IssueImportOption();
- Client client = server.newClient();
- try {
- Set<String> labels = new LinkedHashSet<>();
- for (String each: gitHubRepos) {
- String apiEndpoint = server.getApiEndpoint("/repos/" + each + "/labels");
- for (JsonNode labelNode: list(client, apiEndpoint, logger))
- labels.add(labelNode.get("name").asText());
- }
-
- for (String label: labels) {
- IssueLabelMapping mapping = new IssueLabelMapping();
- mapping.setGitHubIssueLabel(label);
- importOption.getIssueLabelMappings().add(mapping);
- }
- } finally {
- client.close();
- }
- return importOption;
- }
-
- @Nullable
- static User getUser(Client client, ImportServer importSource,
- Map<String, Optional<User>> users, String login, TaskLogger logger) {
- Optional<User> userOpt = users.get(login);
- if (userOpt == null) {
- String apiEndpoint = importSource.getApiEndpoint("/users/" + login);
- String email = get(client, apiEndpoint, logger).get("email").asText(null);
- if (email != null)
- userOpt = Optional.ofNullable(OneDev.getInstance(UserManager.class).findByVerifiedEmailAddress(email));
- else
- userOpt = Optional.empty();
- users.put(login, userOpt);
- }
- return userOpt.orElse(null);
- }
-
- static ImportResult importIssues(ImportServer server, String gitHubRepo, Project oneDevProject,
- boolean useExistingIssueNumbers, IssueImportOption importOption, Map<String, Optional<User>> users,
- boolean dryRun, TaskLogger logger) {
- Client client = server.newClient();
- try {
- Set<String> nonExistentMilestones = new HashSet<>();
- Set<String> nonExistentLogins = new HashSet<>();
- Set<String> unmappedIssueLabels = new HashSet<>();
-
- Map<String, Pair<FieldSpec, String>> labelMappings = new HashMap<>();
- Map<String, Milestone> milestoneMappings = new HashMap<>();
-
- for (IssueLabelMapping mapping: importOption.getIssueLabelMappings()) {
- String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");
- String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");
- FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);
- if (fieldSpec == null)
- throw new ExplicitException("No field spec found: " + oneDevFieldName);
- labelMappings.put(mapping.getGitHubIssueLabel(), new Pair<>(fieldSpec, oneDevFieldValue));
- }
-
- for (Milestone milestone: oneDevProject.getMilestones())
- milestoneMappings.put(milestone.getName(), milestone);
-
- String initialIssueState = getIssueSetting().getInitialStateSpec().getName();
-
- List<Issue> issues = new ArrayList<>();
-
- Map<Long, Long> issueNumberMappings = new HashMap<>();
-
- AtomicInteger numOfImportedIssues = new AtomicInteger(0);
- PageDataConsumer pageDataConsumer = new PageDataConsumer() {
-
- private String joinAsMultilineHtml(List<String> values) {
- List<String> escapedValues = new ArrayList<>();
- for (String value: values)
- escapedValues.add(HtmlEscape.escapeHtml5(value));
- return StringUtils.join(escapedValues, "<br>");
- }
-
- @Override
- public void consume(List<JsonNode> pageData) throws InterruptedException {
- for (JsonNode issueNode: pageData) {
- if (Thread.interrupted())
- throw new InterruptedException();
-
- Map<String, String> extraIssueInfo = new LinkedHashMap<>();
-
- Issue issue = new Issue();
- issue.setProject(oneDevProject);
- issue.setTitle(issueNode.get("title").asText());
- issue.setDescription(issueNode.get("body").asText(null));
- issue.setNumberScope(oneDevProject.getForkRoot());
-
- Long oldNumber = issueNode.get("number").asLong();
- Long newNumber;
- if (dryRun || useExistingIssueNumbers)
- newNumber = oldNumber;
- else
- newNumber = OneDev.getInstance(IssueManager.class).getNextNumber(oneDevProject);
- issue.setNumber(newNumber);
- issueNumberMappings.put(oldNumber, newNumber);
-
- if (issueNode.get("state").asText().equals("closed"))
- issue.setState(importOption.getClosedIssueState());
- else
- issue.setState(initialIssueState);
-
- if (issueNode.hasNonNull("milestone")) {
- String milestoneName = issueNode.get("milestone").get("title").asText();
- Milestone milestone = milestoneMappings.get(milestoneName);
- if (milestone != null) {
- IssueSchedule schedule = new IssueSchedule();
- schedule.setIssue(issue);
- schedule.setMilestone(milestone);
- issue.getSchedules().add(schedule);
- } else {
- extraIssueInfo.put("Milestone", milestoneName);
- nonExistentMilestones.add(milestoneName);
- }
- }
-
- String login = issueNode.get("user").get("login").asText(null);
- User user = getUser(client, server, users, login, logger);
- if (user != null) {
- issue.setSubmitter(user);
- } else {
- issue.setSubmitter(OneDev.getInstance(UserManager.class).getUnknown());
- nonExistentLogins.add(login);
- }
-
- issue.setSubmitDate(ISODateTimeFormat.dateTimeNoMillis()
- .parseDateTime(issueNode.get("created_at").asText())
- .toDate());
-
- LastUpdate lastUpdate = new LastUpdate();
- lastUpdate.setActivity("Opened");
- lastUpdate.setDate(issue.getSubmitDate());
- lastUpdate.setUser(issue.getSubmitter());
- issue.setLastUpdate(lastUpdate);
-
- for (JsonNode assigneeNode: issueNode.get("assignees")) {
- IssueField assigneeField = new IssueField();
- assigneeField.setIssue(issue);
- assigneeField.setName(importOption.getAssigneesIssueField());
- assigneeField.setType(InputSpec.USER);
-
- login = assigneeNode.get("login").asText();
- user = getUser(client, server, users, login, logger);
- if (user != null) {
- assigneeField.setValue(user.getName());
- issue.getFields().add(assigneeField);
- } else {
- nonExistentLogins.add(login);
- }
- }
-
- String apiEndpoint = server.getApiEndpoint("/repos/" + gitHubRepo
- + "/issues/" + oldNumber + "/comments");
- for (JsonNode commentNode: list(client, apiEndpoint, logger)) {
- IssueComment comment = new IssueComment();
- comment.setIssue(issue);
- comment.setContent(commentNode.get("body").asText(null));
- comment.setDate(ISODateTimeFormat.dateTimeNoMillis()
- .parseDateTime(commentNode.get("created_at").asText())
- .toDate());
-
- login = commentNode.get("user").get("login").asText();
- user = getUser(client, server, users, login, logger);
- if (user != null) {
- comment.setUser(user);
- } else {
- comment.setUser(OneDev.getInstance(UserManager.class).getUnknown());
- nonExistentLogins.add(login);
- }
-
- issue.getComments().add(comment);
- }
-
- issue.setCommentCount(issue.getComments().size());
-
- apiEndpoint = server.getApiEndpoint("/repos/" + gitHubRepo
- + "/issues/" + oldNumber + "/labels");
- List<String> currentUnmappedLabels = new ArrayList<>();
- for (JsonNode labelNode: list(client, apiEndpoint, logger)) {
- String labelName = labelNode.get("name").asText();
- Pair<FieldSpec, String> mapped = labelMappings.get(labelName);
- if (mapped != null) {
- IssueField tagField = new IssueField();
- tagField.setIssue(issue);
- tagField.setName(mapped.getFirst().getName());
- tagField.setType(InputSpec.ENUMERATION);
- tagField.setValue(mapped.getSecond());
- tagField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));
- issue.getFields().add(tagField);
- } else {
- currentUnmappedLabels.add(labelName);
- unmappedIssueLabels.add(HtmlEscape.escapeHtml5(labelName));
- }
- }
-
- if (!currentUnmappedLabels.isEmpty())
- extraIssueInfo.put("Labels", joinAsMultilineHtml(currentUnmappedLabels));
-
- Set<String> fieldAndValues = new HashSet<>();
- for (IssueField field: issue.getFields()) {
- String fieldAndValue = field.getName() + "::" + field.getValue();
- if (!fieldAndValues.add(fieldAndValue)) {
- String errorMessage = String.format(
- "Duplicate issue field mapping (issue: %s, field: %s)",
- gitHubRepo + "#" + oldNumber, fieldAndValue);
- throw new ExplicitException(errorMessage);
- }
- }
-
- if (!extraIssueInfo.isEmpty()) {
- StringBuilder builder = new StringBuilder("|");
- for (String key: extraIssueInfo.keySet())
- builder.append(key).append("|");
- builder.append("\n|");
- extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));
- builder.append("\n|");
- for (String value: extraIssueInfo.values())
- builder.append(value).append("|");
-
- if (issue.getDescription() != null)
- issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());
- else
- issue.setDescription(builder.toString());
- }
- issues.add(issue);
- }
- logger.log("Imported " + numOfImportedIssues.addAndGet(pageData.size()) + " issues");
- }
-
- };
-
- String apiEndpoint = server.getApiEndpoint("/repos/" + gitHubRepo + "/issues?state=all&direction=asc");
- list(client, apiEndpoint, pageDataConsumer, logger);
-
- if (!dryRun) {
- ReferenceMigrator migrator = new ReferenceMigrator(Issue.class, issueNumberMappings);
- Dao dao = OneDev.getInstance(Dao.class);
- for (Issue issue: issues) {
- if (issue.getDescription() != null)
- issue.setDescription(migrator.migratePrefixed(issue.getDescription(), "#"));
-
- OneDev.getInstance(IssueManager.class).save(issue);
- for (IssueSchedule schedule: issue.getSchedules())
- dao.persist(schedule);
- for (IssueField field: issue.getFields())
- dao.persist(field);
- for (IssueComment comment: issue.getComments()) {
- comment.setContent(migrator.migratePrefixed(comment.getContent(), "#"));
- dao.persist(comment);
- }
- }
- }
-
- ImportResult result = new ImportResult();
- result.nonExistentLogins.addAll(nonExistentLogins);
- result.nonExistentMilestones.addAll(nonExistentMilestones);
- result.unmappedIssueLabels.addAll(unmappedIssueLabels);
-
- if (numOfImportedIssues.get() != 0)
- result.issuesImported = true;
-
- return result;
- } finally {
- client.close();
- }
- }
-
- static GlobalIssueSetting getIssueSetting() {
- return OneDev.getInstance(SettingManager.class).getIssueSetting();
- }
-
- static List<JsonNode> list(Client client, String apiEndpoint, TaskLogger logger) {
- List<JsonNode> result = new ArrayList<>();
- list(client, apiEndpoint, new PageDataConsumer() {
-
- @Override
- public void consume(List<JsonNode> pageData) {
- result.addAll(pageData);
- }
-
- }, logger);
- return result;
- }
-
- static void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer,
- TaskLogger logger) {
- URI uri;
- try {
- uri = new URIBuilder(apiEndpoint)
- .addParameter("per_page", String.valueOf(PER_PAGE)).build();
- } catch (URISyntaxException e) {
- throw new RuntimeException(e);
- }
-
- int page = 1;
- while (true) {
- try {
- URIBuilder builder = new URIBuilder(uri);
- builder.addParameter("page", String.valueOf(page));
- List<JsonNode> pageData = new ArrayList<>();
- for (JsonNode each: get(client, builder.build().toString(), logger))
- pageData.add(each);
- pageDataConsumer.consume(pageData);
- if (pageData.size() < PER_PAGE)
- break;
- page++;
- } catch (URISyntaxException|InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- static JsonNode get(Client client, String apiEndpoint, TaskLogger logger) {
- WebTarget target = client.target(apiEndpoint);
- Invocation.Builder builder = target.request();
- while (true) {
- try (Response response = builder.get()) {
- int status = response.getStatus();
- if (status != 200) {
- String errorMessage = response.readEntity(String.class);
- if (StringUtils.isNotBlank(errorMessage)) {
- if (errorMessage.contains("rate limit exceeded")) {
- long resetTime = Long.parseLong(response.getHeaderString("x-ratelimit-reset"))*1000L;
- logger.log("Rate limit exceeded, wait until reset...");
- try {
- Thread.sleep(resetTime + 60*1000L - System.currentTimeMillis());
- continue;
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- } else {
- throw new ExplicitException(String.format("Http request failed (url: %s, status code: %d, error message: %s)",
- apiEndpoint, status, errorMessage));
- }
- } else {
- throw new ExplicitException(String.format("Http request failed (status: %s)", status));
- }
- }
- return response.readEntity(JsonNode.class);
- }
- }
- }
-
-}
\ No newline at end of file
diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/IssueImportSource.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/IssueImportSource.java
deleted file mode 100644
index 6524544e0f..0000000000
--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/IssueImportSource.java
+++ /dev/null
@@ -1,75 +0,0 @@
-package io.onedev.server.plugin.imports.github;
-
-import static io.onedev.server.plugin.imports.github.ImportUtils.list;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import javax.ws.rs.client.Client;
-
-import org.hibernate.validator.constraints.NotEmpty;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.databind.JsonNode;
-
-import io.onedev.commons.utils.TaskLogger;
-import io.onedev.server.web.editable.annotation.ChoiceProvider;
-import io.onedev.server.web.editable.annotation.Editable;
-import io.onedev.server.web.util.WicketUtils;
-
-@Editable
-public class IssueImportSource implements Serializable {
-
- private static final long serialVersionUID = 1L;
-
- private static final Logger logger = LoggerFactory.getLogger(IssueImportSource.class);
-
- private String repository;
-
- @Editable(order=100, name="GitHub Repository", description="Choose GitHub repository to "
- + "import issues from. Available repositories will be populated when above options "
- + "are specified")
- @ChoiceProvider("getRepositoryChoices")
- @NotEmpty
- public String getRepository() {
- return repository;
- }
-
- public void setRepository(String repository) {
- this.repository = repository;
- }
-
- @SuppressWarnings("unused")
- private static List<String> getRepositoryChoices() {
- List<String> choices = new ArrayList<>();
-
- ImportServer server = WicketUtils.getPage().getMetaData(ImportServer.META_DATA_KEY);
-
- Client client = server.newClient();
- try {
- String apiEndpoint = server.getApiEndpoint("/user/repos");
- TaskLogger logger = new TaskLogger() {
-
- @Override
- public void log(String message, String sessionId) {
- IssueImportSource.logger.info(message);
- }
-
- };
- for (JsonNode repoNode: list(client, apiEndpoint, logger)) {
- String repoName = repoNode.get("name").asText();
- String ownerName = repoNode.get("owner").get("login").asText();
- choices.add(ownerName + "/" + repoName);
- }
- } finally {
- client.close();
- }
-
- Collections.sort(choices);
- return choices;
- }
-
-}
diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabIssueImporter.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabIssueImporter.java
index 67ca0e8397..825255c647 100644
--- a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabIssueImporter.java
+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabIssueImporter.java
@@ -1,9 +1,8 @@
package io.onedev.server.plugin.imports.gitlab;
-import static io.onedev.server.plugin.imports.gitlab.ImportUtils.NAME;
-import static io.onedev.server.plugin.imports.gitlab.ImportUtils.importIssues;
-
+import java.io.Serializable;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -13,35 +12,82 @@ import io.onedev.commons.utils.TaskLogger;
import io.onedev.server.imports.IssueImporter;
import io.onedev.server.model.Project;
import io.onedev.server.model.User;
-import io.onedev.server.web.util.WicketUtils;
+import io.onedev.server.web.util.ImportStep;
-public class GitLabIssueImporter extends IssueImporter<ImportServer, IssueImportSource, IssueImportOption> {
+public class GitLabIssueImporter implements IssueImporter {
private static final long serialVersionUID = 1L;
+ private final ImportStep<ImportServer> serverStep = new ImportStep<ImportServer>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Authenticate to GitLab";
+ }
+
+ @Override
+ protected ImportServer newSetting() {
+ return new ImportServer();
+ }
+
+ };
+
+ private final ImportStep<ImportProject> projectStep = new ImportStep<ImportProject>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Choose project";
+ }
+
+ @Override
+ protected ImportProject newSetting() {
+ ImportProject repository = new ImportProject();
+ repository.server = serverStep.getSetting();
+ return repository;
+ }
+
+ };
+
+ private final ImportStep<IssueImportOption> optionStep = new ImportStep<IssueImportOption>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Specify import option";
+ }
+
+ @Override
+ protected IssueImportOption newSetting() {
+ return serverStep.getSetting().buildIssueImportOption(
+ Lists.newArrayList(projectStep.getSetting().getProject()));
+ }
+
+ };
+
@Override
public String getName() {
- return NAME;
+ return GitLabPluginModule.NAME;
}
@Override
- public String doImport(ImportServer where, IssueImportSource what, IssueImportOption how, Project project,
- boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {
- logger.log("Importing issues from project " + what.getProject() + "...");
+ public String doImport(Project project, boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {
+ ImportServer server = serverStep.getSetting();
+ String gitLabProject = projectStep.getSetting().getProject();
+ IssueImportOption option = optionStep.getSetting();
+ logger.log("Importing issues from project " + gitLabProject + "...");
Map<String, Optional<User>> users = new HashMap<>();
- return importIssues(where, what.getProject(), project, retainIssueNumbers, how, users, dryRun, logger)
+ return server.importIssues(gitLabProject, project, retainIssueNumbers, option, users, dryRun, logger)
.toHtml("Issues imported successfully");
}
@Override
- public IssueImportSource getWhat(ImportServer where, TaskLogger logger) {
- WicketUtils.getPage().setMetaData(ImportServer.META_DATA_KEY, where);
- return new IssueImportSource();
+ public List<ImportStep<? extends Serializable>> getSteps() {
+ return Lists.newArrayList(serverStep, projectStep, optionStep);
}
- @Override
- public IssueImportOption getHow(ImportServer where, IssueImportSource what, TaskLogger logger) {
- return ImportUtils.buildImportOption(where, Lists.newArrayList(what.getProject()), logger);
- }
-
}
\ No newline at end of file
diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabPluginModule.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabPluginModule.java
index 28cb4dfdc3..f30d856ffe 100644
--- a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabPluginModule.java
+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabPluginModule.java
@@ -1,6 +1,5 @@
package io.onedev.server.plugin.imports.gitlab;
-import java.io.Serializable;
import java.util.Collection;
import com.google.common.collect.Lists;
@@ -17,14 +16,15 @@ import io.onedev.server.imports.ProjectImporterContribution;
*/
public class GitLabPluginModule extends AbstractPluginModule {
- @Override
+ static final String NAME = "GitLab";
+
protected void configure() {
super.configure();
contribute(ProjectImporterContribution.class, new ProjectImporterContribution() {
@Override
- public Collection<ProjectImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> getImporters() {
+ public Collection<ProjectImporter> getImporters() {
return Lists.newArrayList(new GitLabProjectImporter());
}
@@ -38,7 +38,7 @@ public class GitLabPluginModule extends AbstractPluginModule {
contribute(IssueImporterContribution.class, new IssueImporterContribution() {
@Override
- public Collection<IssueImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> getImporters() {
+ public Collection<IssueImporter> getImporters() {
return Lists.newArrayList(new GitLabIssueImporter());
}
diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabProjectImporter.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabProjectImporter.java
index 055a40bad5..459c65c489 100644
--- a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabProjectImporter.java
+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/GitLabProjectImporter.java
@@ -1,173 +1,111 @@
package io.onedev.server.plugin.imports.gitlab;
-import static io.onedev.server.plugin.imports.gitlab.ImportUtils.NAME;
-import static io.onedev.server.plugin.imports.gitlab.ImportUtils.list;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
+import java.io.Serializable;
import java.util.List;
-import java.util.Map;
-import java.util.Optional;
import java.util.stream.Collectors;
-import javax.ws.rs.client.Client;
-
-import org.apache.http.client.utils.URIBuilder;
-import org.joda.time.format.ISODateTimeFormat;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
import io.onedev.commons.utils.TaskLogger;
-import io.onedev.server.OneDev;
-import io.onedev.server.entitymanager.MilestoneManager;
-import io.onedev.server.entitymanager.ProjectManager;
import io.onedev.server.imports.ProjectImporter;
-import io.onedev.server.model.Milestone;
-import io.onedev.server.model.Project;
-import io.onedev.server.model.User;
-import io.onedev.server.storage.StorageManager;
-import io.onedev.server.util.JerseyUtils;
+import io.onedev.server.web.util.ImportStep;
-public class GitLabProjectImporter extends ProjectImporter<ImportServer, ProjectImportSource, ProjectImportOption> {
+public class GitLabProjectImporter implements ProjectImporter {
private static final long serialVersionUID = 1L;
- @Override
- public String getName() {
- return NAME;
- }
-
- private List<Milestone> getMilestones(ImportServer server, String groupId, TaskLogger logger) {
- Client client = server.newClient();
- try {
- List<Milestone> milestones = new ArrayList<>();
- String apiEndpoint = server.getApiEndpoint("/groups/" + groupId + "/milestones");
- for (JsonNode milestoneNode: list(client, apiEndpoint, logger))
- milestones.add(getMilestone(milestoneNode));
- apiEndpoint = server.getApiEndpoint("/groups/" + groupId);
- JsonNode groupNode = JerseyUtils.get(client, apiEndpoint, logger);
- JsonNode parentIdNode = groupNode.get("parent_id");
- if (parentIdNode != null && parentIdNode.asText(null) != null)
- milestones.addAll(getMilestones(server, parentIdNode.asText(), logger));
- return milestones;
- } finally {
- client.close();
+ private final ImportStep<ImportServer> serverStep = new ImportStep<ImportServer>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Authenticate to GitLab";
}
- }
+
+ @Override
+ protected ImportServer newSetting() {
+ return new ImportServer();
+ }
+
+ };
- private Milestone getMilestone(JsonNode milestoneNode) {
- Milestone milestone = new Milestone();
- milestone.setName(milestoneNode.get("title").asText());
- milestone.setDescription(milestoneNode.get("description").asText(null));
- String dueDateString = milestoneNode.get("due_date").asText(null);
- if (dueDateString != null)
- milestone.setDueDate(ISODateTimeFormat.date().parseDateTime(dueDateString).toDate());
- if (milestoneNode.get("state").asText().equals("closed"))
- milestone.setClosed(true);
- return milestone;
- }
+ private final ImportStep<ImportGroup> groupStep = new ImportStep<ImportGroup>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Choose group";
+ }
+
+ @Override
+ protected ImportGroup newSetting() {
+ ImportGroup group = new ImportGroup();
+ group.server = serverStep.getSetting();
+ return group;
+ }
+
+ };
- @Override
- public String doImport(ImportServer where, ProjectImportSource what, ProjectImportOption how,
- boolean dryRun, TaskLogger logger) {
- Collection<Long> projectIds = new ArrayList<>();
- Client client = where.newClient();
- try {
- Map<String, Optional<User>> users = new HashMap<>();
- ImportResult result = new ImportResult();
- for (ProjectMapping projectMapping: what.getProjectMappings()) {
- logger.log("Cloning code from project " + projectMapping.getGitLabProject() + "...");
-
- String apiEndpoint = where.getApiEndpoint("/projects/" + projectMapping.getGitLabProject().replace("/", "%2F"));
- JsonNode projectNode = JerseyUtils.get(client, apiEndpoint, logger);
- ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
- Project project = projectManager.initialize(projectMapping.getOneDevProject());
- Preconditions.checkState(project.isNew());
- project.setDescription(projectNode.get("description").asText(null));
- project.setIssueManagement(projectNode.get("issues_enabled").asBoolean());
-
- String visibility = projectNode.get("visibility").asText();
- if (!visibility.equals("private") && how.getPublicRole() != null)
- project.setDefaultRole(how.getPublicRole());
-
- URIBuilder builder = new URIBuilder(projectNode.get("http_url_to_repo").asText());
- if (!visibility.equals("public"))
- builder.setUserInfo("git", where.getAccessToken());
-
- if (!dryRun) {
- projectManager.clone(project, builder.build().toString());
- projectIds.add(project.getId());
- }
-
- if (how.getIssueImportOption() != null) {
- List<Milestone> milestones = new ArrayList<>();
- logger.log("Importing milestones from project " + projectMapping.getGitLabProject() + "...");
- apiEndpoint = where.getApiEndpoint("/projects/"
- + projectMapping.getGitLabProject().replace("/", "%2F") + "/milestones");
- for (JsonNode milestoneNode: list(client, apiEndpoint, logger))
- milestones.add(getMilestone(milestoneNode));
- JsonNode namespaceNode = projectNode.get("namespace");
- if (namespaceNode.get("kind").asText().equals("group")) {
- String groupId = namespaceNode.get("id").asText();
- milestones.addAll(getMilestones(where, groupId, logger));
- }
-
- for (Milestone milestone: milestones) {
- milestone.setProject(project);
- project.getMilestones().add(milestone);
- if (!dryRun)
- OneDev.getInstance(MilestoneManager.class).save(milestone);
- }
-
- logger.log("Importing issues from project " + projectMapping.getGitLabProject() + "...");
- ImportResult currentResult = ImportUtils.importIssues(where, projectMapping.getGitLabProject(),
- project, true, how.getIssueImportOption(), users, dryRun, logger);
- result.nonExistentLogins.addAll(currentResult.nonExistentLogins);
- result.nonExistentMilestones.addAll(currentResult.nonExistentMilestones);
- result.unmappedIssueLabels.addAll(currentResult.unmappedIssueLabels);
- result.tooLargeAttachments.addAll(currentResult.tooLargeAttachments);
- result.errorAttachments.addAll(currentResult.errorAttachments);
- }
- }
-
- return result.toHtml("Projects imported successfully");
- } catch (Exception e) {
- for (Long projectId: projectIds)
- OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);
- throw new RuntimeException(e);
- } finally {
- client.close();
+ private final ImportStep<ImportProjects> projectsStep = new ImportStep<ImportProjects>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Specify projects";
}
- }
- @Override
- public ProjectImportSource getWhat(ImportServer where, TaskLogger logger) {
- ProjectImportSource importSource = new ProjectImportSource();
- Client client = where.newClient();
- try {
- String apiEndpoint = where.getApiEndpoint("/projects?membership=true");
- for (JsonNode projectNode: list(client, apiEndpoint, logger)) {
- String pathWithNamespace = projectNode.get("path_with_namespace").asText();
+ @Override
+ protected ImportProjects newSetting() {
+ ImportProjects projects = new ImportProjects();
+ String groupId = groupStep.getSetting().getGroupId();
+ for (String project: serverStep.getSetting().listProjects(groupId)) {
ProjectMapping projectMapping = new ProjectMapping();
- projectMapping.setGitLabProject(pathWithNamespace);
- projectMapping.setOneDevProject(pathWithNamespace);
- importSource.getProjectMappings().add(projectMapping);
- }
- } finally {
- client.close();
+ projectMapping.setGitLabProject(project);
+ projectMapping.setOneDevProject(project);
+ projects.getProjectMappings().add(projectMapping);
+ }
+ return projects;
+ }
+
+ };
+
+ private final ImportStep<ProjectImportOption> optionStep = new ImportStep<ProjectImportOption>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Specify import option";
+ }
+
+ @Override
+ protected ProjectImportOption newSetting() {
+ ProjectImportOption option = new ProjectImportOption();
+ List<String> gitLabProjects = projectsStep.getSetting().getProjectMappings().stream()
+ .map(it->it.getGitLabProject()).collect(Collectors.toList());
+ option.setIssueImportOption(serverStep.getSetting().buildIssueImportOption(gitLabProjects));
+ return option;
}
- return importSource;
+
+ };
+
+ @Override
+ public String getName() {
+ return GitLabPluginModule.NAME;
+ }
+
+ @Override
+ public String doImport(boolean dryRun, TaskLogger logger) {
+ ImportServer server = serverStep.getSetting();
+ return server.importProjects(projectsStep.getSetting(), optionStep.getSetting(), dryRun, logger);
}
@Override
- public ProjectImportOption getHow(ImportServer where, ProjectImportSource what, TaskLogger logger) {
- List<String> gitLabProjects = what.getProjectMappings().stream()
- .map(it->it.getGitLabProject()).collect(Collectors.toList());
- ProjectImportOption importOption = new ProjectImportOption();
- importOption.setIssueImportOption(ImportUtils.buildImportOption(where, gitLabProjects, logger));
- return importOption;
+ public List<ImportStep<? extends Serializable>> getSteps() {
+ return Lists.newArrayList(serverStep, groupStep, projectsStep, optionStep);
}
}
\ No newline at end of file
diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportGroup.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportGroup.java
new file mode 100644
index 0000000000..fa9a17a605
--- /dev/null
+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportGroup.java
@@ -0,0 +1,38 @@
+package io.onedev.server.plugin.imports.gitlab;
+
+import java.io.Serializable;
+import java.util.Map;
+
+import io.onedev.server.util.ComponentContext;
+import io.onedev.server.web.editable.BeanEditor;
+import io.onedev.server.web.editable.annotation.ChoiceProvider;
+import io.onedev.server.web.editable.annotation.Editable;
+
+@Editable
+public class ImportGroup implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ ImportServer server;
+
+ private String groupId;
+
+ @Editable(order=100, name="GitLab Group", description="Specify group to import from. "
+ + "Leave empty to import from projects under current account")
+ @ChoiceProvider("getGroupChoices")
+ public String getGroupId() {
+ return groupId;
+ }
+
+ public void setGroupId(String groupId) {
+ this.groupId = groupId;
+ }
+
+ @SuppressWarnings("unused")
+ private static Map<String, String> getGroupChoices() {
+ BeanEditor editor = ComponentContext.get().getComponent().findParent(BeanEditor.class);
+ ImportGroup setting = (ImportGroup) editor.getModelObject();
+ return setting.server.listGroups();
+ }
+
+}
diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportProject.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportProject.java
new file mode 100644
index 0000000000..958e370660
--- /dev/null
+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportProject.java
@@ -0,0 +1,39 @@
+package io.onedev.server.plugin.imports.gitlab;
+
+import java.util.List;
+
+import org.hibernate.validator.constraints.NotEmpty;
+
+import io.onedev.server.util.ComponentContext;
+import io.onedev.server.util.EditContext;
+import io.onedev.server.web.editable.BeanEditor;
+import io.onedev.server.web.editable.annotation.ChoiceProvider;
+import io.onedev.server.web.editable.annotation.Editable;
+
+@Editable
+public class ImportProject extends ImportGroup {
+
+ private static final long serialVersionUID = 1L;
+
+ private String project;
+
+ @Editable(order=200, name="GitLab Project", description="Select project to import from")
+ @ChoiceProvider("getProjectChoices")
+ @NotEmpty
+ public String getProject() {
+ return project;
+ }
+
+ public void setProject(String project) {
+ this.project = project;
+ }
+
+ @SuppressWarnings("unused")
+ private static List<String> getProjectChoices() {
+ BeanEditor editor = ComponentContext.get().getComponent().findParent(BeanEditor.class);
+ ImportProject project = (ImportProject) editor.getModelObject();
+ String groupId = (String) EditContext.get().getInputValue("groupId");
+ return project.server.listProjects(groupId);
+ }
+
+}
diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ProjectImportSource.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportProjects.java
similarity index 93%
rename from server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ProjectImportSource.java
rename to server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportProjects.java
index a5097c04b2..1b16ab120b 100644
--- a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ProjectImportSource.java
+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportProjects.java
@@ -15,16 +15,18 @@ import io.onedev.server.model.Project;
import io.onedev.server.util.validation.Validatable;
import io.onedev.server.util.validation.annotation.ClassValidating;
import io.onedev.server.web.editable.annotation.Editable;
+import io.onedev.server.web.editable.annotation.OmitName;
@Editable
@ClassValidating
-public class ProjectImportSource implements Serializable, Validatable {
+public class ImportProjects implements Serializable, Validatable {
private static final long serialVersionUID = 1L;
private List<ProjectMapping> projectMappings = new ArrayList<>();
- @Editable(order=100, name="Projects to Import")
+ @Editable(order=100)
+ @OmitName
@Size(min=1, max=10000, message="No projects to import")
public List<ProjectMapping> getProjectMappings() {
return projectMappings;
diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportServer.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportServer.java
index fe10e35a14..7d00ba0551 100644
--- a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportServer.java
+++ b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportServer.java
@@ -1,7 +1,25 @@
package io.onedev.server.plugin.imports.gitlab;
+import java.io.IOException;
+import java.io.InputStream;
import java.io.Serializable;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import javax.annotation.Nullable;
import javax.validation.ConstraintValidatorContext;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
@@ -9,13 +27,48 @@ import javax.ws.rs.client.Invocation;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response;
-import org.apache.wicket.MetaDataKey;
+import org.apache.http.client.utils.URIBuilder;
import org.glassfish.jersey.client.ClientProperties;
import org.glassfish.jersey.client.oauth2.OAuth2ClientSupport;
import org.hibernate.validator.constraints.NotEmpty;
+import org.joda.time.format.ISODateTimeFormat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.unbescape.html.HtmlEscape;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.base.Preconditions;
+
+import edu.emory.mathcs.backport.java.util.Collections;
+import io.onedev.commons.utils.ExplicitException;
import io.onedev.commons.utils.StringUtils;
+import io.onedev.commons.utils.TaskLogger;
+import io.onedev.server.OneDev;
+import io.onedev.server.entitymanager.IssueManager;
+import io.onedev.server.entitymanager.MilestoneManager;
+import io.onedev.server.entitymanager.ProjectManager;
+import io.onedev.server.entitymanager.SettingManager;
+import io.onedev.server.entitymanager.UserManager;
+import io.onedev.server.entityreference.ReferenceMigrator;
+import io.onedev.server.model.Issue;
+import io.onedev.server.model.IssueComment;
+import io.onedev.server.model.IssueField;
+import io.onedev.server.model.IssueSchedule;
+import io.onedev.server.model.Milestone;
+import io.onedev.server.model.Project;
+import io.onedev.server.model.User;
+import io.onedev.server.model.support.LastUpdate;
+import io.onedev.server.model.support.administration.GlobalIssueSetting;
+import io.onedev.server.model.support.inputspec.InputSpec;
+import io.onedev.server.model.support.issue.field.spec.FieldSpec;
+import io.onedev.server.persistence.dao.Dao;
+import io.onedev.server.storage.StorageManager;
+import io.onedev.server.util.AttachmentTooLargeException;
+import io.onedev.server.util.CollectionUtils;
+import io.onedev.server.util.DateUtils;
import io.onedev.server.util.JerseyUtils;
+import io.onedev.server.util.JerseyUtils.PageDataConsumer;
+import io.onedev.server.util.Pair;
import io.onedev.server.util.validation.Validatable;
import io.onedev.server.util.validation.annotation.ClassValidating;
import io.onedev.server.web.editable.annotation.Editable;
@@ -27,20 +80,20 @@ public class ImportServer implements Serializable, Validatable {
private static final long serialVersionUID = 1L;
- static final MetaDataKey<ImportServer> META_DATA_KEY = new MetaDataKey<ImportServer>() {
-
- private static final long serialVersionUID = 1L;
-
- };
+ private static final Logger logger = LoggerFactory.getLogger(ImportServer.class);
+
+ private static final int PER_PAGE = 50;
- static final String PROP_API_URL = "apiUrl";
+ private static final Pattern PATTERN_ATTACHMENT = Pattern.compile("\\[(.+?)\\]\\s*\\((/uploads/.+?)\\)");
- static final String PROP_ACCESS_TOKEN = "accessToken";
+ private static final String PROP_API_URL = "apiUrl";
+
+ private static final String PROP_ACCESS_TOKEN = "accessToken";
private String apiUrl;
private String accessToken;
-
+
@Editable(order=10, name="GitLab API URL", description="Specify GitLab API url, for instance <tt>https://gitlab.example.com/api/v4</tt>")
@NotEmpty
public String getApiUrl() {
@@ -62,12 +115,616 @@ public class ImportServer implements Serializable, Validatable {
public void setAccessToken(String accessToken) {
this.accessToken = accessToken;
}
+
+ Map<String, String> listGroups() {
+ Map<String, String> groups = new HashMap<>();
+ Client client = newClient();
+ try {
+ String apiEndpoint = getApiEndpoint("/groups?owned=true&top_level_only=true");
+ for (JsonNode groupNode: list(client, apiEndpoint, new TaskLogger() {
+
+ @Override
+ public void log(String message, String sessionId) {
+ logger.info(message);
+ }
+
+ })) {
+ groups.put(groupNode.get("id").asText(), groupNode.get("full_path").asText());
+ }
+
+ } catch (Exception e) {
+ logger.error("Error listing groups", e);
+ } finally {
+ client.close();
+ }
+
+ return CollectionUtils.sortByValue(groups);
+ }
-
- public String getApiEndpoint(String apiPath) {
+ List<String> listProjects(@Nullable String groupId) {
+ List<String> projects = new ArrayList<>();
+ Client client = newClient();
+ try {
+ String apiEndpoint;
+ if (groupId != null)
+ apiEndpoint = getApiEndpoint("/groups/" + groupId + "/projects?include_subgroups=true");
+ else
+ apiEndpoint = getApiEndpoint("/projects?owned=true");
+ Collection<JsonNode> projectNodes = list(client, apiEndpoint, new TaskLogger() {
+
+ @Override
+ public void log(String message, String sessionId) {
+ logger.info(message);
+ }
+
+ });
+ for (JsonNode projectNode: projectNodes) {
+ if (groupId != null || projectNode.get("namespace").get("kind").asText().equals("user"))
+ projects.add(projectNode.get("path_with_namespace").asText());
+ }
+ } finally {
+ client.close();
+ }
+
+ Collections.sort(projects);
+ return projects;
+ }
+
+ private String getApiEndpoint(String apiPath) {
return StringUtils.stripEnd(apiUrl, "/") + "/" + StringUtils.stripStart(apiPath, "/");
}
+ private Client newClient() {
+ Client client = ClientBuilder.newClient();
+ client.property(ClientProperties.FOLLOW_REDIRECTS, true);
+ client.register(OAuth2ClientSupport.feature(getAccessToken()));
+ return client;
+ }
+
+ IssueImportOption buildIssueImportOption(Collection<String> gitLabProjects) {
+ IssueImportOption option = new IssueImportOption();
+ Client client = newClient();
+ try {
+ TaskLogger taskLogger = new TaskLogger() {
+
+ @Override
+ public void log(String message, String sessionId) {
+ logger.info(message);
+ }
+
+ };
+ Set<String> labels = new LinkedHashSet<>();
+ for (String each: gitLabProjects) {
+ String apiEndpoint = getApiEndpoint("/projects/" + each.replace("/", "%2F") + "/labels");
+ for (JsonNode labelNode: list(client, apiEndpoint, taskLogger))
+ labels.add(labelNode.get("name").asText());
+ }
+
+ for (String label: labels) {
+ IssueLabelMapping mapping = new IssueLabelMapping();
+ mapping.setGitLabIssueLabel(label);
+ option.getIssueLabelMappings().add(mapping);
+ }
+ } finally {
+ client.close();
+ }
+ return option;
+ }
+
+ @Nullable
+ private User getUser(Client client, Map<String, Optional<User>> users,
+ String userId, TaskLogger logger) {
+ Optional<User> userOpt = users.get(userId);
+ if (userOpt == null) {
+ String apiEndpoint = getApiEndpoint("/users/" + userId);
+ JsonNode userNode = JerseyUtils.get(client, apiEndpoint, logger);
+ String email = null;
+ if (userNode.hasNonNull("email"))
+ email = userNode.get("email").asText(null);
+ if (email == null && userNode.hasNonNull("public_email"))
+ email = userNode.get("public_email").asText(null);
+ if (email != null)
+ userOpt = Optional.ofNullable(OneDev.getInstance(UserManager.class).findByVerifiedEmailAddress(email));
+ else
+ userOpt = Optional.empty();
+ users.put(userId, userOpt);
+ }
+ return userOpt.orElse(null);
+ }
+
+ private List<Milestone> getMilestones(String groupId, TaskLogger logger) {
+ Client client = newClient();
+ try {
+ List<Milestone> milestones = new ArrayList<>();
+ String apiEndpoint = getApiEndpoint("/groups/" + groupId + "/milestones");
+ for (JsonNode milestoneNode: list(client, apiEndpoint, logger))
+ milestones.add(getMilestone(milestoneNode));
+ apiEndpoint = getApiEndpoint("/groups/" + groupId);
+ JsonNode groupNode = JerseyUtils.get(client, apiEndpoint, logger);
+ JsonNode parentIdNode = groupNode.get("parent_id");
+ if (parentIdNode != null && parentIdNode.asText(null) != null)
+ milestones.addAll(getMilestones(parentIdNode.asText(), logger));
+ return milestones;
+ } finally {
+ client.close();
+ }
+ }
+
+ private Milestone getMilestone(JsonNode milestoneNode) {
+ Milestone milestone = new Milestone();
+ milestone.setName(milestoneNode.get("title").asText());
+ milestone.setDescription(milestoneNode.get("description").asText(null));
+ String dueDateString = milestoneNode.get("due_date").asText(null);
+ if (dueDateString != null)
+ milestone.setDueDate(ISODateTimeFormat.date().parseDateTime(dueDateString).toDate());
+ if (milestoneNode.get("state").asText().equals("closed"))
+ milestone.setClosed(true);
+ return milestone;
+ }
+
+ String importProjects(ImportProjects projects, ProjectImportOption option, boolean dryRun, TaskLogger logger) {
+ Collection<Long> projectIds = new ArrayList<>();
+ Client client = newClient();
+ try {
+ Map<String, Optional<User>> users = new HashMap<>();
+ ImportResult result = new ImportResult();
+ for (ProjectMapping projectMapping: projects.getProjectMappings()) {
+ logger.log("Cloning code from project " + projectMapping.getGitLabProject() + "...");
+
+ String apiEndpoint = getApiEndpoint("/projects/" + projectMapping.getGitLabProject().replace("/", "%2F"));
+ JsonNode projectNode = JerseyUtils.get(client, apiEndpoint, logger);
+ ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
+ Project project = projectManager.initialize(projectMapping.getOneDevProject());
+ Preconditions.checkState(project.isNew());
+ project.setDescription(projectNode.get("description").asText(null));
+ project.setIssueManagement(projectNode.get("issues_enabled").asBoolean());
+
+ String visibility = projectNode.get("visibility").asText();
+ if (!visibility.equals("private") && option.getPublicRole() != null)
+ project.setDefaultRole(option.getPublicRole());
+
+ URIBuilder builder = new URIBuilder(projectNode.get("http_url_to_repo").asText());
+ if (!visibility.equals("public"))
+ builder.setUserInfo("git", getAccessToken());
+
+ if (!dryRun) {
+ projectManager.clone(project, builder.build().toString());
+ projectIds.add(project.getId());
+ }
+
+ if (option.getIssueImportOption() != null) {
+ List<Milestone> milestones = new ArrayList<>();
+ logger.log("Importing milestones from project " + projectMapping.getGitLabProject() + "...");
+ apiEndpoint = getApiEndpoint("/projects/"
+ + projectMapping.getGitLabProject().replace("/", "%2F") + "/milestones");
+ for (JsonNode milestoneNode: list(client, apiEndpoint, logger))
+ milestones.add(getMilestone(milestoneNode));
+ JsonNode namespaceNode = projectNode.get("namespace");
+ if (namespaceNode.get("kind").asText().equals("group")) {
+ String groupId = namespaceNode.get("id").asText();
+ milestones.addAll(getMilestones(groupId, logger));
+ }
+
+ for (Milestone milestone: milestones) {
+ milestone.setProject(project);
+ project.getMilestones().add(milestone);
+ if (!dryRun)
+ OneDev.getInstance(MilestoneManager.class).save(milestone);
+ }
+
+ logger.log("Importing issues from project " + projectMapping.getGitLabProject() + "...");
+ ImportResult currentResult = importIssues(projectMapping.getGitLabProject(),
+ project, true, option.getIssueImportOption(), users, dryRun, logger);
+ result.nonExistentLogins.addAll(currentResult.nonExistentLogins);
+ result.nonExistentMilestones.addAll(currentResult.nonExistentMilestones);
+ result.unmappedIssueLabels.addAll(currentResult.unmappedIssueLabels);
+ result.tooLargeAttachments.addAll(currentResult.tooLargeAttachments);
+ result.errorAttachments.addAll(currentResult.errorAttachments);
+ }
+ }
+
+ return result.toHtml("Projects imported successfully");
+ } catch (Exception e) {
+ for (Long projectId: projectIds)
+ OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);
+ throw new RuntimeException(e);
+ } finally {
+ client.close();
+ }
+ }
+
+ ImportResult importIssues(String gitLabProject, Project oneDevProject,
+ boolean useExistingIssueNumbers, IssueImportOption option,
+ Map<String, Optional<User>> users, boolean dryRun, TaskLogger logger) {
+ Client client = newClient();
+ try {
+ Set<String> nonExistentMilestones = new HashSet<>();
+ Set<String> nonExistentLogins = new HashSet<>();
+ Set<String> unmappedIssueLabels = new HashSet<>();
+ Set<String> tooLargeAttachments = new LinkedHashSet<>();
+ Set<String> errorAttachments = new HashSet<>();
+
+ Map<String, Pair<FieldSpec, String>> labelMappings = new HashMap<>();
+ Map<String, Milestone> milestoneMappings = new HashMap<>();
+
+ for (IssueLabelMapping mapping: option.getIssueLabelMappings()) {
+ String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");
+ String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");
+ FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);
+ if (fieldSpec == null)
+ throw new ExplicitException("No field spec found: " + oneDevFieldName);
+ labelMappings.put(mapping.getGitLabIssueLabel(), new Pair<>(fieldSpec, oneDevFieldValue));
+ }
+
+ for (Milestone milestone: oneDevProject.getMilestones())
+ milestoneMappings.put(milestone.getName(), milestone);
+
+ String initialIssueState = getIssueSetting().getInitialStateSpec().getName();
+
+ List<Issue> issues = new ArrayList<>();
+
+ Map<Long, Long> issueNumberMappings = new HashMap<>();
+
+ AtomicInteger numOfImportedIssues = new AtomicInteger(0);
+ PageDataConsumer pageDataConsumer = new PageDataConsumer() {
+
+ @Nullable
+ private String processAttachments(String issueUUID, String issueFQN, String markdown,
+ String attachmentRootUrl, Set<String> tooLargeAttachments) {
+ StringBuffer buffer = new StringBuffer();
+ Matcher matcher = PATTERN_ATTACHMENT.matcher(markdown);
+ while (matcher.find()) {
+ String attachmentUrl = attachmentRootUrl + matcher.group(2);
+ String attachmentName = StringUtils.substringAfterLast(attachmentUrl, "/");
+ WebTarget target = client.target(attachmentUrl);
+ Invocation.Builder builder = target.request();
+ try (Response response = builder.get()) {
+ String errorMessage = JerseyUtils.checkStatus(attachmentUrl, response);
+ if (errorMessage != null) {
+ logger.error("Error downloading attachment: " + errorMessage);
+ errorAttachments.add(attachmentUrl);
+ } else {
+ try (InputStream is = response.readEntity(InputStream.class)) {
+ String oneDevAttachmentName = oneDevProject.saveAttachment(issueUUID, attachmentName, is);
+ String oneDevAttachmentUrl = oneDevProject.getAttachmentUrlPath(issueUUID, oneDevAttachmentName);
+ matcher.appendReplacement(buffer,
+ Matcher.quoteReplacement("[" + matcher.group(1) + "](" + oneDevAttachmentUrl + ")"));
+ } catch (AttachmentTooLargeException ex) {
+ tooLargeAttachments.add(issueFQN + ":" + matcher.group(2));
+ } catch (IOException e) {
+ logger.error("Error downloading attachment", e);
+ errorAttachments.add(attachmentUrl);
+ }
+ }
+ }
+ }
+ matcher.appendTail(buffer);
+
+ return buffer.toString();
+ }
+
+ private String joinAsMultilineHtml(List<String> values) {
+ List<String> escapedValues = new ArrayList<>();
+ for (String value: values)
+ escapedValues.add(HtmlEscape.escapeHtml5(value));
+ return StringUtils.join(escapedValues, "<br>");
+ }
+
+ @Override
+ public void consume(List<JsonNode> pageData) throws InterruptedException {
+ for (JsonNode issueNode: pageData) {
+ if (Thread.interrupted())
+ throw new InterruptedException();
+
+ Map<String, String> extraIssueInfo = new LinkedHashMap<>();
+
+ Issue issue = new Issue();
+ issue.setProject(oneDevProject);
+ issue.setTitle(issueNode.get("title").asText());
+ issue.setDescription(issueNode.get("description").asText(null));
+
+ issue.setNumberScope(oneDevProject.getForkRoot());
+
+ Long oldNumber = issueNode.get("iid").asLong();
+ Long newNumber;
+ if (dryRun || useExistingIssueNumbers)
+ newNumber = oldNumber;
+ else
+ newNumber = OneDev.getInstance(IssueManager.class).getNextNumber(oneDevProject);
+ issue.setNumber(newNumber);
+ issueNumberMappings.put(oldNumber, newNumber);
+
+ String issueFQN = gitLabProject + "#" + oldNumber;
+
+ if (issueNode.get("state").asText().equals("closed"))
+ issue.setState(option.getClosedIssueState());
+ else
+ issue.setState(initialIssueState);
+
+ if (issueNode.hasNonNull("milestone")) {
+ String milestoneName = issueNode.get("milestone").get("title").asText();
+ Milestone milestone = milestoneMappings.get(milestoneName);
+ if (milestone != null) {
+ IssueSchedule schedule = new IssueSchedule();
+ schedule.setIssue(issue);
+ schedule.setMilestone(milestone);
+ issue.getSchedules().add(schedule);
+ } else {
+ extraIssueInfo.put("Milestone", milestoneName);
+ nonExistentMilestones.add(milestoneName);
+ }
+ }
+
+ JsonNode authorNode = issueNode.get("author");
+ User user = getUser(client, users, authorNode.get("id").asText(), logger);
+ if (user != null) {
+ issue.setSubmitter(user);
+ } else {
+ issue.setSubmitter(OneDev.getInstance(UserManager.class).getUnknown());
+ nonExistentLogins.add(authorNode.get("username").asText());
+ }
+
+ issue.setSubmitDate(ISODateTimeFormat.dateTime()
+ .parseDateTime(issueNode.get("created_at").asText())
+ .toDate());
+
+ LastUpdate lastUpdate = new LastUpdate();
+ lastUpdate.setActivity("Opened");
+ lastUpdate.setDate(issue.getSubmitDate());
+ lastUpdate.setUser(issue.getSubmitter());
+ issue.setLastUpdate(lastUpdate);
+
+ List<JsonNode> assigneeNodes = new ArrayList<>();
+ if (issueNode.hasNonNull("assignees")) {
+ for (JsonNode assigneeNode: issueNode.get("assignees"))
+ assigneeNodes.add(assigneeNode);
+ } else if (issueNode.hasNonNull("assignee")) {
+ assigneeNodes.add(issueNode.get("assignee"));
+ }
+
+ for (JsonNode assigneeNode: assigneeNodes) {
+ IssueField assigneeField = new IssueField();
+ assigneeField.setIssue(issue);
+ assigneeField.setName(option.getAssigneesIssueField());
+ assigneeField.setType(InputSpec.USER);
+
+ user = getUser(client, users, assigneeNode.get("id").asText(), logger);
+ if (user != null) {
+ assigneeField.setValue(user.getName());
+ issue.getFields().add(assigneeField);
+ } else {
+ nonExistentLogins.add(assigneeNode.get("username").asText());
+ }
+ }
+
+ if (option.getDueDateIssueField() != null) {
+ String dueDate = issueNode.get("due_date").asText(null);
+ if (dueDate != null) {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(option.getDueDateIssueField());
+ issueField.setType(InputSpec.DATE);
+ issueField.setValue(dueDate);
+ issue.getFields().add(issueField);
+ }
+ }
+
+ JsonNode timeStatsNode = issueNode.get("time_stats");
+ if (option.getEstimatedTimeIssueField() != null) {
+ int value = timeStatsNode.get("time_estimate").asInt();
+ if (value != 0) {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(option.getEstimatedTimeIssueField());
+ issueField.setType(InputSpec.WORKING_PERIOD);
+ issueField.setValue(DateUtils.formatWorkingPeriod(value/60));
+ issue.getFields().add(issueField);
+ }
+ }
+ if (option.getSpentTimeIssueField() != null) {
+ int value = timeStatsNode.get("total_time_spent").asInt();
+ if (value != 0) {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(option.getSpentTimeIssueField());
+ issueField.setType(InputSpec.WORKING_PERIOD);
+ issueField.setValue(DateUtils.formatWorkingPeriod(value/60));
+ issue.getFields().add(issueField);
+ }
+ }
+
+ List<String> currentUnmappedLabels = new ArrayList<>();
+ for (JsonNode labelNode: issueNode.get("labels")) {
+ String labelName = labelNode.asText();
+ Pair<FieldSpec, String> mapped = labelMappings.get(labelName);
+ if (mapped != null) {
+ IssueField labelField = new IssueField();
+ labelField.setIssue(issue);
+ labelField.setName(mapped.getFirst().getName());
+ labelField.setType(InputSpec.ENUMERATION);
+ labelField.setValue(mapped.getSecond());
+ labelField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));
+ issue.getFields().add(labelField);
+ } else {
+ currentUnmappedLabels.add(labelName);
+ unmappedIssueLabels.add(HtmlEscape.escapeHtml5(labelName));
+ }
+ }
+
+ if (!currentUnmappedLabels.isEmpty())
+ extraIssueInfo.put("Labels", joinAsMultilineHtml(currentUnmappedLabels));
+
+ String webUrl = issueNode.get("web_url").asText();
+ String attachmentRootUrl = StringUtils.substringBeforeLast(webUrl, "/-");
+ if (!dryRun && issue.getDescription() != null) {
+ issue.setDescription(processAttachments(issue.getUUID(), issueFQN,
+ issue.getDescription(), attachmentRootUrl, tooLargeAttachments));
+ }
+
+ String apiEndpoint = getApiEndpoint("/projects/" + gitLabProject.replace("/", "%2F")
+ + "/issues/" + oldNumber + "/links");
+ Map<String, List<String>> links = new LinkedHashMap<>();
+ for (JsonNode linkNode: list(client, apiEndpoint, logger)) {
+ String linkIssueNumber = linkNode.get("iid").asText();
+ if (linkNode.get("references").get("full").asText().equals(gitLabProject + "#" + linkIssueNumber)) {
+ String linkType = linkNode.get("link_type").asText();
+ List<String> linksOfType = links.get(linkType);
+ if (linksOfType == null) {
+ linksOfType = new ArrayList<>();
+ links.put(linkType, linksOfType);
+ }
+ linksOfType.add("#" + linkIssueNumber);
+ }
+ }
+ for (Map.Entry<String, List<String>> entry: links.entrySet())
+ extraIssueInfo.put(entry.getKey(), joinAsMultilineHtml(entry.getValue()));
+
+ apiEndpoint = getApiEndpoint("/projects/" + gitLabProject.replace("/", "%2F")
+ + "/issues/" + oldNumber + "/notes?sort=asc");
+ for (JsonNode noteNode: list(client, apiEndpoint, logger)) {
+ if (!noteNode.get("system").asBoolean()) {
+ String commentContent = noteNode.get("body").asText(null);
+ if (commentContent != null) {
+ if (!dryRun) {
+ commentContent = processAttachments(issue.getUUID(), issueFQN,
+ commentContent, attachmentRootUrl, tooLargeAttachments);
+ }
+
+ IssueComment comment = new IssueComment();
+ comment.setIssue(issue);
+ comment.setContent(commentContent);
+ comment.setDate(ISODateTimeFormat.dateTime()
+ .parseDateTime(noteNode.get("created_at").asText())
+ .toDate());
+
+ authorNode = noteNode.get("author");
+ user = getUser(client, users, authorNode.get("id").asText(), logger);
+ if (user != null) {
+ comment.setUser(user);
+ } else {
+ comment.setUser(OneDev.getInstance(UserManager.class).getUnknown());
+ nonExistentLogins.add(authorNode.get("username").asText());
+ }
+ issue.getComments().add(comment);
+ }
+ }
+ }
+
+ issue.setCommentCount(issue.getComments().size());
+
+ Set<String> fieldAndValues = new HashSet<>();
+ for (IssueField field: issue.getFields()) {
+ String fieldAndValue = field.getName() + "::" + field.getValue();
+ if (!fieldAndValues.add(fieldAndValue)) {
+ String errorMessage = String.format(
+ "Duplicate issue field mapping (issue: %s, field: %s)",
+ issueFQN, fieldAndValue);
+ throw new ExplicitException(errorMessage);
+ }
+ }
+
+ if (!extraIssueInfo.isEmpty()) {
+ StringBuilder builder = new StringBuilder("|");
+ for (String key: extraIssueInfo.keySet())
+ builder.append(key).append("|");
+ builder.append("\n|");
+ extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));
+ builder.append("\n|");
+ for (String value: extraIssueInfo.values())
+ builder.append(value).append("|");
+
+ if (issue.getDescription() != null)
+ issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());
+ else
+ issue.setDescription(builder.toString());
+ }
+ issues.add(issue);
+ }
+ logger.log("Imported " + numOfImportedIssues.addAndGet(pageData.size()) + " issues");
+ }
+
+ };
+
+ String apiEndpoint = getApiEndpoint("/projects/" + gitLabProject.replace("/", "%2F") + "/issues?sort=asc");
+ list(client, apiEndpoint, pageDataConsumer, logger);
+
+ if (!dryRun) {
+ ReferenceMigrator migrator = new ReferenceMigrator(Issue.class, issueNumberMappings);
+ Dao dao = OneDev.getInstance(Dao.class);
+ for (Issue issue: issues) {
+ if (issue.getDescription() != null)
+ issue.setDescription(migrator.migratePrefixed(issue.getDescription(), "#"));
+
+ OneDev.getInstance(IssueManager.class).save(issue);
+ for (IssueSchedule schedule: issue.getSchedules())
+ dao.persist(schedule);
+ for (IssueField field: issue.getFields())
+ dao.persist(field);
+ for (IssueComment comment: issue.getComments()) {
+ comment.setContent(migrator.migratePrefixed(comment.getContent(), "#"));
+ dao.persist(comment);
+ }
+ }
+ }
+
+ ImportResult result = new ImportResult();
+ result.nonExistentLogins.addAll(nonExistentLogins);
+ result.nonExistentMilestones.addAll(nonExistentMilestones);
+ result.unmappedIssueLabels.addAll(unmappedIssueLabels);
+ result.tooLargeAttachments.addAll(tooLargeAttachments);
+ result.errorAttachments.addAll(errorAttachments);
+
+ return result;
+ } finally {
+ client.close();
+ }
+ }
+
+ private GlobalIssueSetting getIssueSetting() {
+ return OneDev.getInstance(SettingManager.class).getIssueSetting();
+ }
+
+ List<JsonNode> list(Client client, String apiEndpoint, TaskLogger logger) {
+ List<JsonNode> result = new ArrayList<>();
+ list(client, apiEndpoint, new PageDataConsumer() {
+
+ @Override
+ public void consume(List<JsonNode> pageData) {
+ result.addAll(pageData);
+ }
+
+ }, logger);
+ return result;
+ }
+
+ private void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer,
+ TaskLogger logger) {
+ URI uri;
+ try {
+ uri = new URIBuilder(apiEndpoint)
+ .addParameter("per_page", String.valueOf(PER_PAGE)).build();
+ } catch (URISyntaxException e) {
+ throw new RuntimeException(e);
+ }
+
+ int page = 1;
+ while (true) {
+ try {
+ URIBuilder builder = new URIBuilder(uri);
+ builder.addParameter("page", String.valueOf(page));
+ List<JsonNode> pageData = new ArrayList<>();
+ for (JsonNode each: JerseyUtils.get(client, builder.build().toString(), logger))
+ pageData.add(each);
+ pageDataConsumer.consume(pageData);
+ if (pageData.size() < PER_PAGE)
+ break;
+ page++;
+ } catch (URISyntaxException|InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
@Override
public boolean isValid(ConstraintValidatorContext context) {
Client client = ClientBuilder.newClient();
@@ -113,11 +770,4 @@ public class ImportServer implements Serializable, Validatable {
return true;
}
- Client newClient() {
- Client client = ClientBuilder.newClient();
- client.property(ClientProperties.FOLLOW_REDIRECTS, true);
- client.register(OAuth2ClientSupport.feature(getAccessToken()));
- return client;
- }
-
}
diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportUtils.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportUtils.java
deleted file mode 100644
index 6dfc1f45f1..0000000000
--- a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/ImportUtils.java
+++ /dev/null
@@ -1,502 +0,0 @@
-package io.onedev.server.plugin.imports.gitlab;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import javax.annotation.Nullable;
-import javax.ws.rs.client.Client;
-import javax.ws.rs.client.Invocation;
-import javax.ws.rs.client.WebTarget;
-import javax.ws.rs.core.Response;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.http.client.utils.URIBuilder;
-import org.joda.time.format.ISODateTimeFormat;
-import org.unbescape.html.HtmlEscape;
-
-import com.fasterxml.jackson.databind.JsonNode;
-
-import io.onedev.commons.utils.ExplicitException;
-import io.onedev.commons.utils.TaskLogger;
-import io.onedev.server.OneDev;
-import io.onedev.server.entitymanager.IssueManager;
-import io.onedev.server.entitymanager.SettingManager;
-import io.onedev.server.entitymanager.UserManager;
-import io.onedev.server.entityreference.ReferenceMigrator;
-import io.onedev.server.model.Issue;
-import io.onedev.server.model.IssueComment;
-import io.onedev.server.model.IssueField;
-import io.onedev.server.model.IssueSchedule;
-import io.onedev.server.model.Milestone;
-import io.onedev.server.model.Project;
-import io.onedev.server.model.User;
-import io.onedev.server.model.support.LastUpdate;
-import io.onedev.server.model.support.administration.GlobalIssueSetting;
-import io.onedev.server.model.support.inputspec.InputSpec;
-import io.onedev.server.model.support.issue.field.spec.FieldSpec;
-import io.onedev.server.persistence.dao.Dao;
-import io.onedev.server.util.AttachmentTooLargeException;
-import io.onedev.server.util.DateUtils;
-import io.onedev.server.util.JerseyUtils;
-import io.onedev.server.util.JerseyUtils.PageDataConsumer;
-import io.onedev.server.util.Pair;
-
-public class ImportUtils {
-
- static final String NAME = "GitLab";
-
- static final int PER_PAGE = 50;
-
- private static final Pattern PATTERN_ATTACHMENT = Pattern.compile("\\[(.+?)\\]\\s*\\((/uploads/.+?)\\)");
-
- static IssueImportOption buildImportOption(ImportServer server, Collection<String> gitLabProjects, TaskLogger logger) {
- IssueImportOption importOption = new IssueImportOption();
- Client client = server.newClient();
- try {
- Set<String> labels = new LinkedHashSet<>();
- for (String each: gitLabProjects) {
- String apiEndpoint = server.getApiEndpoint("/projects/" + each.replace("/", "%2F") + "/labels");
- for (JsonNode labelNode: list(client, apiEndpoint, logger))
- labels.add(labelNode.get("name").asText());
- }
-
- for (String label: labels) {
- IssueLabelMapping mapping = new IssueLabelMapping();
- mapping.setGitLabIssueLabel(label);
- importOption.getIssueLabelMappings().add(mapping);
- }
- } finally {
- client.close();
- }
- return importOption;
- }
-
- @Nullable
- static User getUser(Client client, ImportServer importSource, Map<String, Optional<User>> users,
- String userId, TaskLogger logger) {
- Optional<User> userOpt = users.get(userId);
- if (userOpt == null) {
- String apiEndpoint = importSource.getApiEndpoint("/users/" + userId);
- JsonNode userNode = JerseyUtils.get(client, apiEndpoint, logger);
- String email = null;
- if (userNode.hasNonNull("email"))
- email = userNode.get("email").asText(null);
- if (email == null && userNode.hasNonNull("public_email"))
- email = userNode.get("public_email").asText(null);
- if (email != null)
- userOpt = Optional.ofNullable(OneDev.getInstance(UserManager.class).findByVerifiedEmailAddress(email));
- else
- userOpt = Optional.empty();
- users.put(userId, userOpt);
- }
- return userOpt.orElse(null);
- }
-
- static ImportResult importIssues(ImportServer server, String gitLabProject, Project oneDevProject,
- boolean useExistingIssueNumbers, IssueImportOption importOption, Map<String, Optional<User>> users,
- boolean dryRun, TaskLogger logger) {
- Client client = server.newClient();
- try {
- Set<String> nonExistentMilestones = new HashSet<>();
- Set<String> nonExistentLogins = new HashSet<>();
- Set<String> unmappedIssueLabels = new HashSet<>();
- Set<String> tooLargeAttachments = new LinkedHashSet<>();
- Set<String> errorAttachments = new HashSet<>();
-
- Map<String, Pair<FieldSpec, String>> labelMappings = new HashMap<>();
- Map<String, Milestone> milestoneMappings = new HashMap<>();
-
- for (IssueLabelMapping mapping: importOption.getIssueLabelMappings()) {
- String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");
- String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");
- FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);
- if (fieldSpec == null)
- throw new ExplicitException("No field spec found: " + oneDevFieldName);
- labelMappings.put(mapping.getGitLabIssueLabel(), new Pair<>(fieldSpec, oneDevFieldValue));
- }
-
- for (Milestone milestone: oneDevProject.getMilestones())
- milestoneMappings.put(milestone.getName(), milestone);
-
- String initialIssueState = getIssueSetting().getInitialStateSpec().getName();
-
- List<Issue> issues = new ArrayList<>();
-
- Map<Long, Long> issueNumberMappings = new HashMap<>();
-
- AtomicInteger numOfImportedIssues = new AtomicInteger(0);
- PageDataConsumer pageDataConsumer = new PageDataConsumer() {
-
- @Nullable
- private String processAttachments(String issueUUID, String issueFQN, String markdown,
- String attachmentRootUrl, Set<String> tooLargeAttachments) {
- StringBuffer buffer = new StringBuffer();
- Matcher matcher = PATTERN_ATTACHMENT.matcher(markdown);
- while (matcher.find()) {
- String attachmentUrl = attachmentRootUrl + matcher.group(2);
- String attachmentName = StringUtils.substringAfterLast(attachmentUrl, "/");
- WebTarget target = client.target(attachmentUrl);
- Invocation.Builder builder = target.request();
- try (Response response = builder.get()) {
- String errorMessage = JerseyUtils.checkStatus(attachmentUrl, response);
- if (errorMessage != null) {
- logger.error("Error downloading attachment: " + errorMessage);
- errorAttachments.add(attachmentUrl);
- } else {
- try (InputStream is = response.readEntity(InputStream.class)) {
- String oneDevAttachmentName = oneDevProject.saveAttachment(issueUUID, attachmentName, is);
- String oneDevAttachmentUrl = oneDevProject.getAttachmentUrlPath(issueUUID, oneDevAttachmentName);
- matcher.appendReplacement(buffer,
- Matcher.quoteReplacement("[" + matcher.group(1) + "](" + oneDevAttachmentUrl + ")"));
- } catch (AttachmentTooLargeException ex) {
- tooLargeAttachments.add(issueFQN + ":" + matcher.group(2));
- } catch (IOException e) {
- logger.error("Error downloading attachment", e);
- errorAttachments.add(attachmentUrl);
- }
- }
- }
- }
- matcher.appendTail(buffer);
-
- return buffer.toString();
- }
-
- private String joinAsMultilineHtml(List<String> values) {
- List<String> escapedValues = new ArrayList<>();
- for (String value: values)
- escapedValues.add(HtmlEscape.escapeHtml5(value));
- return StringUtils.join(escapedValues, "<br>");
- }
-
- @Override
- public void consume(List<JsonNode> pageData) throws InterruptedException {
- for (JsonNode issueNode: pageData) {
- if (Thread.interrupted())
- throw new InterruptedException();
-
- Map<String, String> extraIssueInfo = new LinkedHashMap<>();
-
- Issue issue = new Issue();
- issue.setProject(oneDevProject);
- issue.setTitle(issueNode.get("title").asText());
- issue.setDescription(issueNode.get("description").asText(null));
-
- issue.setNumberScope(oneDevProject.getForkRoot());
-
- Long oldNumber = issueNode.get("iid").asLong();
- Long newNumber;
- if (dryRun || useExistingIssueNumbers)
- newNumber = oldNumber;
- else
- newNumber = OneDev.getInstance(IssueManager.class).getNextNumber(oneDevProject);
- issue.setNumber(newNumber);
- issueNumberMappings.put(oldNumber, newNumber);
-
- String issueFQN = gitLabProject + "#" + oldNumber;
-
- if (issueNode.get("state").asText().equals("closed"))
- issue.setState(importOption.getClosedIssueState());
- else
- issue.setState(initialIssueState);
-
- if (issueNode.hasNonNull("milestone")) {
- String milestoneName = issueNode.get("milestone").get("title").asText();
- Milestone milestone = milestoneMappings.get(milestoneName);
- if (milestone != null) {
- IssueSchedule schedule = new IssueSchedule();
- schedule.setIssue(issue);
- schedule.setMilestone(milestone);
- issue.getSchedules().add(schedule);
- } else {
- extraIssueInfo.put("Milestone", milestoneName);
- nonExistentMilestones.add(milestoneName);
- }
- }
-
- JsonNode authorNode = issueNode.get("author");
- User user = getUser(client, server, users, authorNode.get("id").asText(), logger);
- if (user != null) {
- issue.setSubmitter(user);
- } else {
- issue.setSubmitter(OneDev.getInstance(UserManager.class).getUnknown());
- nonExistentLogins.add(authorNode.get("username").asText());
- }
-
- issue.setSubmitDate(ISODateTimeFormat.dateTime()
- .parseDateTime(issueNode.get("created_at").asText())
- .toDate());
-
- LastUpdate lastUpdate = new LastUpdate();
- lastUpdate.setActivity("Opened");
- lastUpdate.setDate(issue.getSubmitDate());
- lastUpdate.setUser(issue.getSubmitter());
- issue.setLastUpdate(lastUpdate);
-
- List<JsonNode> assigneeNodes = new ArrayList<>();
- if (issueNode.hasNonNull("assignees")) {
- for (JsonNode assigneeNode: issueNode.get("assignees"))
- assigneeNodes.add(assigneeNode);
- } else if (issueNode.hasNonNull("assignee")) {
- assigneeNodes.add(issueNode.get("assignee"));
- }
-
- for (JsonNode assigneeNode: assigneeNodes) {
- IssueField assigneeField = new IssueField();
- assigneeField.setIssue(issue);
- assigneeField.setName(importOption.getAssigneesIssueField());
- assigneeField.setType(InputSpec.USER);
-
- user = getUser(client, server, users, assigneeNode.get("id").asText(), logger);
- if (user != null) {
- assigneeField.setValue(user.getName());
- issue.getFields().add(assigneeField);
- } else {
- nonExistentLogins.add(assigneeNode.get("username").asText());
- }
- }
-
- if (importOption.getDueDateIssueField() != null) {
- String dueDate = issueNode.get("due_date").asText(null);
- if (dueDate != null) {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(importOption.getDueDateIssueField());
- issueField.setType(InputSpec.DATE);
- issueField.setValue(dueDate);
- issue.getFields().add(issueField);
- }
- }
-
- JsonNode timeStatsNode = issueNode.get("time_stats");
- if (importOption.getEstimatedTimeIssueField() != null) {
- int value = timeStatsNode.get("time_estimate").asInt();
- if (value != 0) {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(importOption.getEstimatedTimeIssueField());
- issueField.setType(InputSpec.WORKING_PERIOD);
- issueField.setValue(DateUtils.formatWorkingPeriod(value/60));
- issue.getFields().add(issueField);
- }
- }
- if (importOption.getSpentTimeIssueField() != null) {
- int value = timeStatsNode.get("total_time_spent").asInt();
- if (value != 0) {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(importOption.getSpentTimeIssueField());
- issueField.setType(InputSpec.WORKING_PERIOD);
- issueField.setValue(DateUtils.formatWorkingPeriod(value/60));
- issue.getFields().add(issueField);
- }
- }
-
- List<String> currentUnmappedLabels = new ArrayList<>();
- for (JsonNode labelNode: issueNode.get("labels")) {
- String labelName = labelNode.asText();
- Pair<FieldSpec, String> mapped = labelMappings.get(labelName);
- if (mapped != null) {
- IssueField labelField = new IssueField();
- labelField.setIssue(issue);
- labelField.setName(mapped.getFirst().getName());
- labelField.setType(InputSpec.ENUMERATION);
- labelField.setValue(mapped.getSecond());
- labelField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));
- issue.getFields().add(labelField);
- } else {
- currentUnmappedLabels.add(labelName);
- unmappedIssueLabels.add(HtmlEscape.escapeHtml5(labelName));
- }
- }
-
- if (!currentUnmappedLabels.isEmpty())
- extraIssueInfo.put("Labels", joinAsMultilineHtml(currentUnmappedLabels));
-
- String webUrl = issueNode.get("web_url").asText();
- String attachmentRootUrl = StringUtils.substringBeforeLast(webUrl, "/-");
- if (!dryRun && issue.getDescription() != null) {
- issue.setDescription(processAttachments(issue.getUUID(), issueFQN,
- issue.getDescription(), attachmentRootUrl, tooLargeAttachments));
- }
-
- String apiEndpoint = server.getApiEndpoint("/projects/" + gitLabProject.replace("/", "%2F")
- + "/issues/" + oldNumber + "/links");
- Map<String, List<String>> links = new LinkedHashMap<>();
- for (JsonNode linkNode: list(client, apiEndpoint, logger)) {
- String linkIssueNumber = linkNode.get("iid").asText();
- if (linkNode.get("references").get("full").asText().equals(gitLabProject + "#" + linkIssueNumber)) {
- String linkType = linkNode.get("link_type").asText();
- List<String> linksOfType = links.get(linkType);
- if (linksOfType == null) {
- linksOfType = new ArrayList<>();
- links.put(linkType, linksOfType);
- }
- linksOfType.add("#" + linkIssueNumber);
- }
- }
- for (Map.Entry<String, List<String>> entry: links.entrySet())
- extraIssueInfo.put(entry.getKey(), joinAsMultilineHtml(entry.getValue()));
-
- apiEndpoint = server.getApiEndpoint("/projects/" + gitLabProject.replace("/", "%2F")
- + "/issues/" + oldNumber + "/notes?sort=asc");
- for (JsonNode noteNode: list(client, apiEndpoint, logger)) {
- if (!noteNode.get("system").asBoolean()) {
- String commentContent = noteNode.get("body").asText(null);
- if (commentContent != null) {
- if (!dryRun) {
- commentContent = processAttachments(issue.getUUID(), issueFQN,
- commentContent, attachmentRootUrl, tooLargeAttachments);
- }
-
- IssueComment comment = new IssueComment();
- comment.setIssue(issue);
- comment.setContent(commentContent);
- comment.setDate(ISODateTimeFormat.dateTime()
- .parseDateTime(noteNode.get("created_at").asText())
- .toDate());
-
- authorNode = noteNode.get("author");
- user = getUser(client, server, users, authorNode.get("id").asText(), logger);
- if (user != null) {
- comment.setUser(user);
- } else {
- comment.setUser(OneDev.getInstance(UserManager.class).getUnknown());
- nonExistentLogins.add(authorNode.get("username").asText());
- }
- issue.getComments().add(comment);
- }
- }
- }
-
- issue.setCommentCount(issue.getComments().size());
-
- Set<String> fieldAndValues = new HashSet<>();
- for (IssueField field: issue.getFields()) {
- String fieldAndValue = field.getName() + "::" + field.getValue();
- if (!fieldAndValues.add(fieldAndValue)) {
- String errorMessage = String.format(
- "Duplicate issue field mapping (issue: %s, field: %s)",
- issueFQN, fieldAndValue);
- throw new ExplicitException(errorMessage);
- }
- }
-
- if (!extraIssueInfo.isEmpty()) {
- StringBuilder builder = new StringBuilder("|");
- for (String key: extraIssueInfo.keySet())
- builder.append(key).append("|");
- builder.append("\n|");
- extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));
- builder.append("\n|");
- for (String value: extraIssueInfo.values())
- builder.append(value).append("|");
-
- if (issue.getDescription() != null)
- issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());
- else
- issue.setDescription(builder.toString());
- }
- issues.add(issue);
- }
- logger.log("Imported " + numOfImportedIssues.addAndGet(pageData.size()) + " issues");
- }
-
- };
-
- String apiEndpoint = server.getApiEndpoint("/projects/" + gitLabProject.replace("/", "%2F") + "/issues?sort=asc");
- list(client, apiEndpoint, pageDataConsumer, logger);
-
- if (!dryRun) {
- ReferenceMigrator migrator = new ReferenceMigrator(Issue.class, issueNumberMappings);
- Dao dao = OneDev.getInstance(Dao.class);
- for (Issue issue: issues) {
- if (issue.getDescription() != null)
- issue.setDescription(migrator.migratePrefixed(issue.getDescription(), "#"));
-
- OneDev.getInstance(IssueManager.class).save(issue);
- for (IssueSchedule schedule: issue.getSchedules())
- dao.persist(schedule);
- for (IssueField field: issue.getFields())
- dao.persist(field);
- for (IssueComment comment: issue.getComments()) {
- comment.setContent(migrator.migratePrefixed(comment.getContent(), "#"));
- dao.persist(comment);
- }
- }
- }
-
- ImportResult result = new ImportResult();
- result.nonExistentLogins.addAll(nonExistentLogins);
- result.nonExistentMilestones.addAll(nonExistentMilestones);
- result.unmappedIssueLabels.addAll(unmappedIssueLabels);
- result.tooLargeAttachments.addAll(tooLargeAttachments);
- result.errorAttachments.addAll(errorAttachments);
-
- return result;
- } finally {
- client.close();
- }
- }
-
- static GlobalIssueSetting getIssueSetting() {
- return OneDev.getInstance(SettingManager.class).getIssueSetting();
- }
-
- static List<JsonNode> list(Client client, String apiEndpoint, TaskLogger logger) {
- List<JsonNode> result = new ArrayList<>();
- list(client, apiEndpoint, new PageDataConsumer() {
-
- @Override
- public void consume(List<JsonNode> pageData) {
- result.addAll(pageData);
- }
-
- }, logger);
- return result;
- }
-
- static void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer,
- TaskLogger logger) {
- URI uri;
- try {
- uri = new URIBuilder(apiEndpoint)
- .addParameter("per_page", String.valueOf(PER_PAGE)).build();
- } catch (URISyntaxException e) {
- throw new RuntimeException(e);
- }
-
- int page = 1;
- while (true) {
- try {
- URIBuilder builder = new URIBuilder(uri);
- builder.addParameter("page", String.valueOf(page));
- List<JsonNode> pageData = new ArrayList<>();
- for (JsonNode each: JerseyUtils.get(client, builder.build().toString(), logger))
- pageData.add(each);
- pageDataConsumer.consume(pageData);
- if (pageData.size() < PER_PAGE)
- break;
- page++;
- } catch (URISyntaxException|InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
-}
\ No newline at end of file
diff --git a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/IssueImportSource.java b/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/IssueImportSource.java
deleted file mode 100644
index 3647471264..0000000000
--- a/server-plugin/server-plugin-import-gitlab/src/main/java/io/onedev/server/plugin/imports/gitlab/IssueImportSource.java
+++ /dev/null
@@ -1,69 +0,0 @@
-package io.onedev.server.plugin.imports.gitlab;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
-import javax.ws.rs.client.Client;
-
-import org.hibernate.validator.constraints.NotEmpty;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.databind.JsonNode;
-
-import io.onedev.commons.utils.TaskLogger;
-import io.onedev.server.web.editable.annotation.ChoiceProvider;
-import io.onedev.server.web.editable.annotation.Editable;
-import io.onedev.server.web.util.WicketUtils;
-
-@Editable
-public class IssueImportSource implements Serializable {
-
- private static final long serialVersionUID = 1L;
-
- private static final Logger logger = LoggerFactory.getLogger(IssueImportSource.class);
-
- private String project;
-
- @Editable(order=400, name="GitLab Project", description="Choose GitLab project to import issues from")
- @ChoiceProvider("getProjectChoices")
- @NotEmpty
- public String getProject() {
- return project;
- }
-
- public void setProject(String project) {
- this.project = project;
- }
-
- @SuppressWarnings("unused")
- private static List<String> getProjectChoices() {
- List<String> choices = new ArrayList<>();
-
- ImportServer server = WicketUtils.getPage().getMetaData(ImportServer.META_DATA_KEY);
-
- Client client = server.newClient();
- try {
- String apiEndpoint = server.getApiEndpoint("/projects?membership=true");
- Collection<JsonNode> projectNodes = ImportUtils.list(client, apiEndpoint, new TaskLogger() {
-
- @Override
- public void log(String message, String sessionId) {
- logger.info(message);
- }
-
- });
- for (JsonNode projectNode: projectNodes)
- choices.add(projectNode.get("path_with_namespace").asText());
- } finally {
- client.close();
- }
-
- Collections.sort(choices);
- return choices;
- }
-
-}
diff --git a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportProject.java b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportProject.java
new file mode 100644
index 0000000000..29f08275fd
--- /dev/null
+++ b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportProject.java
@@ -0,0 +1,40 @@
+package io.onedev.server.plugin.imports.jiracloud;
+
+import java.io.Serializable;
+import java.util.List;
+
+import org.hibernate.validator.constraints.NotEmpty;
+
+import io.onedev.server.util.ComponentContext;
+import io.onedev.server.web.editable.BeanEditor;
+import io.onedev.server.web.editable.annotation.ChoiceProvider;
+import io.onedev.server.web.editable.annotation.Editable;
+
+@Editable
+public class ImportProject implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ ImportServer server;
+
+ private String project;
+
+ @Editable(order=400, name="JIRA Project", description="Choose JIRA project to import issues from")
+ @ChoiceProvider("getProjectChoices")
+ @NotEmpty
+ public String getProject() {
+ return project;
+ }
+
+ public void setProject(String project) {
+ this.project = project;
+ }
+
+ @SuppressWarnings("unused")
+ private static List<String> getProjectChoices() {
+ BeanEditor editor = ComponentContext.get().getComponent().findParent(BeanEditor.class);
+ ImportProject setting = (ImportProject) editor.getModelObject();
+ return setting.server.listProjects();
+ }
+
+}
diff --git a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ProjectImportSource.java b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportProjects.java
similarity index 94%
rename from server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ProjectImportSource.java
rename to server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportProjects.java
index f6c09c8339..ea95f4d782 100644
--- a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ProjectImportSource.java
+++ b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportProjects.java
@@ -15,16 +15,18 @@ import io.onedev.server.model.Project;
import io.onedev.server.util.validation.Validatable;
import io.onedev.server.util.validation.annotation.ClassValidating;
import io.onedev.server.web.editable.annotation.Editable;
+import io.onedev.server.web.editable.annotation.OmitName;
@Editable
@ClassValidating
-public class ProjectImportSource implements Serializable, Validatable {
+public class ImportProjects implements Serializable, Validatable {
private static final long serialVersionUID = 1L;
private List<ProjectMapping> projectMappings = new ArrayList<>();
@Editable(order=100, name="Projects to Import")
+ @OmitName
@Size(min=1, max=10000, message="No projects to import")
public List<ProjectMapping> getProjectMappings() {
return projectMappings;
diff --git a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportServer.java b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportServer.java
index 6248fcf98b..c8f1c4a8e6 100644
--- a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportServer.java
+++ b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportServer.java
@@ -1,7 +1,28 @@
package io.onedev.server.plugin.imports.jiracloud;
+import java.io.IOException;
+import java.io.InputStream;
import java.io.Serializable;
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.annotation.Nullable;
import javax.validation.ConstraintValidatorContext;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
@@ -10,14 +31,46 @@ import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
-import org.apache.wicket.MetaDataKey;
+import org.apache.http.client.utils.URIBuilder;
import org.glassfish.jersey.client.ClientProperties;
import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
import org.hibernate.validator.constraints.Email;
import org.hibernate.validator.constraints.NotEmpty;
+import org.joda.time.format.ISODateTimeFormat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.unbescape.html.HtmlEscape;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
+
+import io.onedev.commons.utils.ExplicitException;
import io.onedev.commons.utils.StringUtils;
+import io.onedev.commons.utils.TaskLogger;
+import io.onedev.server.OneDev;
+import io.onedev.server.entitymanager.IssueManager;
+import io.onedev.server.entitymanager.ProjectManager;
+import io.onedev.server.entitymanager.SettingManager;
+import io.onedev.server.entitymanager.UserManager;
+import io.onedev.server.entityreference.ReferenceMigrator;
+import io.onedev.server.model.Issue;
+import io.onedev.server.model.IssueComment;
+import io.onedev.server.model.IssueField;
+import io.onedev.server.model.IssueSchedule;
+import io.onedev.server.model.Project;
+import io.onedev.server.model.User;
+import io.onedev.server.model.support.LastUpdate;
+import io.onedev.server.model.support.administration.GlobalIssueSetting;
+import io.onedev.server.model.support.inputspec.InputSpec;
+import io.onedev.server.model.support.issue.field.spec.FieldSpec;
+import io.onedev.server.persistence.dao.Dao;
+import io.onedev.server.storage.StorageManager;
+import io.onedev.server.util.DateUtils;
import io.onedev.server.util.JerseyUtils;
+import io.onedev.server.util.JerseyUtils.PageDataConsumer;
+import io.onedev.server.util.Pair;
import io.onedev.server.util.validation.Validatable;
import io.onedev.server.util.validation.annotation.ClassValidating;
import io.onedev.server.web.editable.annotation.Editable;
@@ -29,13 +82,11 @@ public class ImportServer implements Serializable, Validatable {
private static final long serialVersionUID = 1L;
- static final MetaDataKey<ImportServer> META_DATA_KEY = new MetaDataKey<ImportServer>() {
-
- private static final long serialVersionUID = 1L;
-
- };
+ private static final Logger logger = LoggerFactory.getLogger(ImportServer.class);
+
+ private static final int PER_PAGE = 50;
- static final String PROP_API_URL = "apiUrl";
+ private static final String PROP_API_URL = "apiUrl";
private String apiUrl;
@@ -75,10 +126,869 @@ public class ImportServer implements Serializable, Validatable {
this.apiToken = apiToken;
}
- public String getApiEndpoint(String apiPath) {
+ private String getApiEndpoint(String apiPath) {
return StringUtils.stripEnd(apiUrl, "/") + "/" + StringUtils.stripStart(apiPath, "/");
}
+
+ List<String> listProjects() {
+ List<String> projects = new ArrayList<>();
+
+ Client client = newClient();
+ try {
+ String apiEndpoint = getApiEndpoint("/project/search");
+ for (JsonNode projectNode: list(client, apiEndpoint, new TaskLogger() {
+
+ @Override
+ public void log(String message, String sessionId) {
+ logger.info(message);
+ }
+
+ })) {
+ projects.add(projectNode.get("name").asText());
+ }
+ } finally {
+ client.close();
+ }
+
+ Collections.sort(projects);
+ return projects;
+ }
+
+ private Client newClient() {
+ Client client = ClientBuilder.newClient();
+ client.property(ClientProperties.FOLLOW_REDIRECTS, true);
+ client.register(HttpAuthenticationFeature.basic(getAccountEmail(), getApiToken()));
+ return client;
+ }
+
+ private GlobalIssueSetting getIssueSetting() {
+ return OneDev.getInstance(SettingManager.class).getIssueSetting();
+ }
+
+ @Nullable
+ private User getUser(Map<String, Optional<User>> users, JsonNode userNode, TaskLogger logger) {
+ String accountId = userNode.get("accountId").asText();
+ Optional<User> userOpt = users.get(accountId);
+ if (userOpt == null) {
+ String displayName = null;
+ if (userNode.hasNonNull("displayName"))
+ displayName = userNode.get("displayName").asText(null);
+ if (displayName != null)
+ userOpt = Optional.ofNullable(OneDev.getInstance(UserManager.class).findByFullName(displayName));
+ else
+ userOpt = Optional.empty();
+ users.put(accountId, userOpt);
+ }
+ return userOpt.orElse(null);
+ }
+
+ private String getMarkdown(JsonNode contentNode, boolean escape) {
+ StringBuilder markdown = new StringBuilder();
+ for (JsonNode itemNode: contentNode) {
+ String type = itemNode.get("type").asText();
+ switch (type) {
+ case "text":
+ String text = itemNode.get("text").asText();
+ if (escape)
+ text = HtmlEscape.escapeHtml5(text);
+
+ JsonNode marksNode = itemNode.get("marks");
+ if (marksNode != null) {
+ for (JsonNode markNode: marksNode) {
+ String textType = markNode.get("type").asText(null);
+ switch (textType) {
+ case "strong":
+ text = "**" + text.replace("*", "\\*") + "**";
+ break;
+ case "em":
+ text = "_" + text.replace("_", "\\_") + "_";
+ break;
+ case "code":
+ text = "`" + text + "`";
+ break;
+ case "textColor":
+ text = String.format("<span style='color:%s'>%s</span>",
+ markNode.get("attrs").get("color").asText(), text);
+ break;
+ case "link":
+ text = String.format("[%s](%s)", text, markNode.get("attrs").get("href").asText());
+ break;
+ }
+ }
+ }
+ markdown.append(text);
+ break;
+ case "mention":
+ markdown.append("@" + itemNode.get("attrs").get("text").asText());
+ break;
+ case "heading":
+ markdown.append(StringUtils.repeat('#', itemNode.get("attrs").get("level").asInt())).append(" ");
+ markdown.append(getMarkdown(itemNode.get("content"), true)).append("\n\n");
+ break;
+ case "paragraph":
+ markdown.append(getMarkdown(itemNode.get("content"), true)).append("\n\n");
+ break;
+ case "bulletList":
+ for (JsonNode listItemNode: itemNode.get("content"))
+ markdown.append("- ").append(getMarkdown(listItemNode.get("content"), true));
+ break;
+ case "orderedList":
+ for (JsonNode listItemNode: itemNode.get("content"))
+ markdown.append("1. ").append(getMarkdown(listItemNode.get("content"), true));
+ break;
+ case "inlineCard":
+ markdown.append(" " + StringUtils.substringAfterLast(itemNode.get("attrs").get("url").asText(), "/") + " ");
+ break;
+ case "emoji":
+ markdown.append(" " + itemNode.get("attrs").get("shortName").asText() + " ");
+ break;
+ case "codeBlock":
+ markdown.append("```\n" + getMarkdown(itemNode.get("content"), false) + "\n```\n\n");
+ break;
+ case "panel":
+ String panelType = itemNode.get("attrs").get("panelType").asText();
+ switch (panelType) {
+ case "info":
+ case "note":
+ markdown.append(":information_source:\n");
+ break;
+ case "success":
+ markdown.append(":white_check_mark:\n");
+ break;
+ case "warning":
+ markdown.append(":warning:\n");
+ break;
+ case "error":
+ markdown.append(":no_entry:\n");
+ break;
+ }
+ markdown.append(getMarkdown(itemNode.get("content"), true));
+ break;
+ case "blockquote":
+ List<String> lines = Splitter.on('\n').splitToList(getMarkdown(itemNode.get("content"), true));
+ for (String line: lines)
+ markdown.append("> ").append(line).append("\n");
+ break;
+ case "rule":
+ markdown.append("---\n");
+ break;
+ case "date":
+ markdown.append(DateUtils.formatDate(new Date(itemNode.get("attrs").get("timestamp").asLong())));
+ break;
+ case "status":
+ JsonNode attrsNode = itemNode.get("attrs");
+ text = HtmlEscape.escapeHtml5(attrsNode.get("text").asText());
+ String color = attrsNode.get("color").asText();
+ switch (color) {
+ case "yellow":
+ markdown.append(String.format("<span style='background:#FFF0B3; padding:2px 4px; border-radius:3px; font-weight:bold; color:#172B4D;'>%s</span>", text));
+ break;
+ case "neutral":
+ markdown.append(String.format("<span style='background:#DFE1E6; padding:2px 4px; border-radius:3px; font-weight:bold; color:#42526E;'>%s</span>", text));
+ break;
+ case "purple":
+ markdown.append(String.format("<span style='background:#EAE6FF; padding:2px 4px; border-radius:3px; font-weight:bold; color:#574BA3;'>%s</span>", text));
+ break;
+ case "blue":
+ markdown.append(String.format("<span style='background:#DEEBFF; padding:2px 4px; border-radius:3px; font-weight:bold; color:#0747A6;'>%s</span>", text));
+ break;
+ case "red":
+ markdown.append(String.format("<span style='background:#FFEBE6; padding:2px 4px; border-radius:3px; font-weight:bold; color:#BF2600;'>%s</span>", text));
+ break;
+ case "green":
+ markdown.append(String.format("<span style='background:#E3FCEF; padding:2px 4px; border-radius:3px; font-weight:bold; color:#006644;'>%s</span>", text));
+ break;
+ default:
+ markdown.append(text);
+ }
+ break;
+ case "table":
+ List<List<String>> tableData = new ArrayList<>();
+ for (JsonNode rowNode: itemNode.get("content")) {
+ List<String> columns = new ArrayList<>();
+ for (JsonNode columnNode: rowNode.get("content"))
+ columns.add(getMarkdown(columnNode.get("content"), true).replace("\n", "<br>"));
+ tableData.add(columns);
+ }
+ if (!tableData.isEmpty()) {
+ if (!tableData.get(0).isEmpty()) {
+ markdown.append("|");
+ for (int i=0; i<tableData.get(0).size(); i++)
+ markdown.append("-----|");
+ markdown.append("\n");
+ for (List<String> row: tableData) {
+ markdown.append("|");
+ for (String column: row)
+ markdown.append(" ").append(column).append(" |");
+ markdown.append("\n");
+ }
+ markdown.append("\n");
+ }
+ }
+ break;
+ }
+ }
+ return markdown.toString();
+ }
+
+ String importProjects(ImportProjects projects, ImportOption option, boolean dryRun, TaskLogger logger) {
+ Collection<Long> projectIds = new ArrayList<>();
+
+ Map<String, JsonNode> projectNodes = getProjectNodes(logger);
+
+ Client client = newClient();
+ try {
+ Map<String, Optional<User>> users = new HashMap<>();
+ ImportResult result = new ImportResult();
+
+ for (ProjectMapping projectMapping: projects.getProjectMappings()) {
+ String jiraProject = projectMapping.getJiraProject();
+ JsonNode projectNode = projectNodes.get(jiraProject);
+ if (projectNode == null)
+ throw new ExplicitException("Unable to find project: " + jiraProject);
+
+ String apiEndpoint = getApiEndpoint("/project/" + projectNode.get("id").asText());
+
+ // Get more detail project information
+ projectNode = JerseyUtils.get(client, apiEndpoint, logger);
+
+ ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
+ Project project = projectManager.initialize(projectMapping.getOneDevProject());
+ Preconditions.checkState(project.isNew());
+
+ project.setDescription(projectNode.get("description").asText(null));
+
+ if (!dryRun) {
+ OneDev.getInstance(ProjectManager.class).create(project);
+ projectIds.add(project.getId());
+ }
+
+ logger.log("Importing issues from project " + jiraProject + "...");
+ ImportResult currentResult = importIssues(projectNode, project, true, option, users, dryRun, logger);
+ result.nonExistentLogins.addAll(currentResult.nonExistentLogins);
+ result.errorAttachments.addAll(currentResult.errorAttachments);
+ result.tooLargeAttachments.addAll(currentResult.tooLargeAttachments);
+ result.unmappedIssuePriorities.addAll(currentResult.unmappedIssuePriorities);
+ result.unmappedIssueStatuses.addAll(currentResult.unmappedIssueStatuses);
+ result.unmappedIssueTypes.addAll(currentResult.unmappedIssueTypes);
+ }
+
+ return result.toHtml("Projects imported successfully");
+ } catch (Exception e) {
+ for (Long projectId: projectIds)
+ OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);
+ throw new RuntimeException(e);
+ } finally {
+ client.close();
+ }
+ }
+
+ String importIssues(Project project, String jiraProject, ImportOption option,
+ boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {
+ Map<String, JsonNode> projectNodes = getProjectNodes(logger);
+
+ Client client = newClient();
+ try {
+ Map<String, Optional<User>> users = new HashMap<>();
+ JsonNode projectNode = projectNodes.get(jiraProject);
+ if (projectNode == null)
+ throw new ExplicitException("Unable to find project: " + jiraProject);
+ ImportResult result = importIssues(projectNode, project, retainIssueNumbers,
+ option, users, dryRun, logger);
+ return result.toHtml("Issues imported successfully");
+ } finally {
+ client.close();
+ }
+ }
+
+ private ImportResult importIssues(JsonNode jiraProject, Project oneDevProject,
+ boolean retainIssueNumbers, ImportOption option,
+ Map<String, Optional<User>> users, boolean dryRun, TaskLogger logger) {
+ Client client = newClient();
+ try {
+ Set<String> unmappedIssueStatuses = new HashSet<>();
+ Set<String> unmappedIssueTypes = new HashSet<>();
+ Set<String> unmappedIssuePriorities = new HashSet<>();
+ Set<String> nonExistentLogins = new HashSet<>();
+ Set<String> tooLargeAttachments = new LinkedHashSet<>();
+ Set<String> errorAttachments = new HashSet<>();
+
+ Map<String, Pair<FieldSpec, String>> typeMappings = new HashMap<>();
+
+ for (IssueTypeMapping mapping: option.getIssueTypeMappings()) {
+ String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");
+ String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");
+ FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);
+ if (fieldSpec == null)
+ throw new ExplicitException("No field spec found: " + oneDevFieldName);
+ typeMappings.put(mapping.getJiraIssueType(), new Pair<>(fieldSpec, oneDevFieldValue));
+ }
+
+ Map<String, Pair<FieldSpec, String>> priorityMappings = new HashMap<>();
+
+ for (IssuePriorityMapping mapping: option.getIssuePriorityMappings()) {
+ String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");
+ String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");
+ FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);
+ if (fieldSpec == null)
+ throw new ExplicitException("No field spec found: " + oneDevFieldName);
+ priorityMappings.put(mapping.getJiraIssuePriority(), new Pair<>(fieldSpec, oneDevFieldValue));
+ }
+
+ String initialIssueState = getIssueSetting().getInitialStateSpec().getName();
+
+ List<Issue> issues = new ArrayList<>();
+
+ Map<Long, Long> issueNumberMappings = new HashMap<>();
+
+ Map<String, String> statusMappings = new HashMap<>();
+ for (IssueStatusMapping mapping: option.getIssueStatusMappings())
+ statusMappings.put(mapping.getJiraIssueStatus(), mapping.getOneDevIssueState());
+
+ Map<String, String> untranslatedStatusNames = new HashMap<>();
+ String apiEndpoint = getApiEndpoint("/status");
+ for (JsonNode statusNode: list(client, apiEndpoint, logger)) {
+ String untranslated = statusNode.get("untranslatedName").asText();
+ untranslatedStatusNames.put(statusNode.get("id").asText(), untranslated);
+ }
+
+ Map<String, String> untranslatedTypeNames = new HashMap<>();
+ apiEndpoint = getApiEndpoint("/issuetype/project?projectId=" + jiraProject.get("id").asText());
+ for (JsonNode typeNode: list(client, apiEndpoint, logger))
+ untranslatedTypeNames.put(typeNode.get("id").asText(), typeNode.get("untranslatedName").asText());
+
+ AtomicInteger numOfImportedIssues = new AtomicInteger(0);
+ PageDataConsumer pageDataConsumer = new PageDataConsumer() {
+
+ private String joinAsMultilineHtml(List<String> values) {
+ List<String> escapedValues = new ArrayList<>();
+ for (String value: values)
+ escapedValues.add(HtmlEscape.escapeHtml5(value));
+ return StringUtils.join(escapedValues, "<br>");
+ }
+
+ @Override
+ public void consume(List<JsonNode> pageData) throws InterruptedException {
+ for (JsonNode issueNode: pageData) {
+ if (Thread.interrupted())
+ throw new InterruptedException();
+
+ String issueKey = issueNode.get("key").asText();
+ String apiEndpoint = getApiEndpoint("/issue/" + issueKey);
+ issueNode = JerseyUtils.get(client, apiEndpoint, logger);
+
+ Map<String, String> extraIssueInfo = new LinkedHashMap<>();
+
+ Issue issue = new Issue();
+ issue.setProject(oneDevProject);
+
+ issue.setNumberScope(oneDevProject.getForkRoot());
+
+ Long oldNumber = Long.valueOf(StringUtils.substringAfterLast(issueNode.get("key").asText(), "-"));
+ Long newNumber;
+ if (dryRun || retainIssueNumbers)
+ newNumber = oldNumber;
+ else
+ newNumber = OneDev.getInstance(IssueManager.class).getNextNumber(oneDevProject);
+ issue.setNumber(newNumber);
+ issueNumberMappings.put(oldNumber, newNumber);
+
+ JsonNode fieldsNode = issueNode.get("fields");
+
+ issue.setTitle(fieldsNode.get("summary").asText());
+ String description = "";
+ JsonNode descriptionNode = fieldsNode.get("description");
+ if (descriptionNode != null && descriptionNode.hasNonNull("content")) {
+ description = getMarkdown(descriptionNode.get("content"), true);
+ if (StringUtils.isNotBlank("description"))
+ description = "#### Description\n\n" + description;
+ }
+
+ JsonNode environmentNode = fieldsNode.get("environment");
+ if (environmentNode != null && environmentNode.hasNonNull("content")) {
+ String environment = getMarkdown(environmentNode.get("content"), true);
+ if (StringUtils.isNotBlank(environment)) {
+ if (StringUtils.isNotBlank(description))
+ description += "\n";
+ description += "#### Environment\n\n" + environment;
+ }
+ }
+
+ if (StringUtils.isNotBlank(description))
+ issue.setDescription(description);
+
+ String statusId = fieldsNode.get("status").get("id").asText();
+ String untranslatedStatusName = untranslatedStatusNames.get(statusId);
+ if (untranslatedStatusName != null) {
+ String statusName = statusMappings.get(untranslatedStatusName);
+ if (statusName != null) {
+ issue.setState(statusName);
+ } else {
+ unmappedIssueStatuses.add(untranslatedStatusName);
+ issue.setState(initialIssueState);
+ }
+ } else {
+ throw new ExplicitException("Can not find JIRA status of id: " + statusId);
+ }
+
+ String typeId = fieldsNode.get("issuetype").get("id").asText();
+ String untranslatedTypeName = untranslatedTypeNames.get(typeId);
+ if (untranslatedTypeName != null) {
+ Pair<FieldSpec, String> mapped = typeMappings.get(untranslatedTypeName);
+ if (mapped != null) {
+ IssueField typeField = new IssueField();
+ typeField.setIssue(issue);
+ typeField.setName(mapped.getFirst().getName());
+ typeField.setType(InputSpec.ENUMERATION);
+ typeField.setValue(mapped.getSecond());
+ typeField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));
+ issue.getFields().add(typeField);
+ } else {
+ extraIssueInfo.put("Type", HtmlEscape.escapeHtml5(untranslatedTypeName));
+ unmappedIssueTypes.add(untranslatedTypeName);
+ }
+ } else {
+ String errorMessage = String.format("Can not find JIRA issue type by id (project: %s, issue type id: %s)",
+ jiraProject.get("name").asText(), typeId);
+ throw new ExplicitException(errorMessage);
+ }
+
+ String priorityName = fieldsNode.get("priority").get("name").asText();
+ Pair<FieldSpec, String> mapped = priorityMappings.get(priorityName);
+ if (mapped != null) {
+ IssueField priorityField = new IssueField();
+ priorityField.setIssue(issue);
+ priorityField.setName(mapped.getFirst().getName());
+ priorityField.setType(InputSpec.ENUMERATION);
+ priorityField.setValue(mapped.getSecond());
+ priorityField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));
+ issue.getFields().add(priorityField);
+ } else {
+ extraIssueInfo.put("Priority", HtmlEscape.escapeHtml5(priorityName));
+ unmappedIssuePriorities.add(priorityName);
+ }
+
+ List<String> components = new ArrayList<>();
+ for (JsonNode componentNode: fieldsNode.get("components"))
+ components.add(componentNode.get("name").asText());
+ if (!components.isEmpty())
+ extraIssueInfo.put("Components", joinAsMultilineHtml(components));
+
+ List<String> affectedVersions = new ArrayList<>();
+ for (JsonNode versionNode: fieldsNode.get("versions"))
+ affectedVersions.add(versionNode.get("name").asText());
+ if (!affectedVersions.isEmpty())
+ extraIssueInfo.put("Affected Versions", joinAsMultilineHtml(affectedVersions));
+
+ List<String> fixVersions = new ArrayList<>();
+ for (JsonNode versionNode: fieldsNode.get("fixVersions"))
+ fixVersions.add(versionNode.get("name").asText());
+ if (!fixVersions.isEmpty())
+ extraIssueInfo.put("Fix Versions", joinAsMultilineHtml(fixVersions));
+
+ if (fieldsNode.hasNonNull("labels")) {
+ List<String> labels = new ArrayList<>();
+ for (JsonNode labelNode: fieldsNode.get("labels"))
+ labels.add(labelNode.asText());
+ if (!labels.isEmpty())
+ extraIssueInfo.put("Labels", joinAsMultilineHtml(labels));
+ }
+
+ JsonNode authorNode = fieldsNode.get("reporter");
+ if (authorNode == null)
+ authorNode = fieldsNode.get("creator");
+ if (authorNode != null) {
+ User user = getUser(users, authorNode, logger);
+ if (user != null) {
+ issue.setSubmitter(user);
+ } else {
+ issue.setSubmitter(OneDev.getInstance(UserManager.class).getUnknown());
+ nonExistentLogins.add(authorNode.get("displayName").asText());
+ }
+ } else {
+ issue.setSubmitter(OneDev.getInstance(UserManager.class).getUnknown());
+ }
+
+ if (fieldsNode.hasNonNull("assignee")) {
+ JsonNode assigneeNode = fieldsNode.get("assignee");
+ IssueField assigneeField = new IssueField();
+ assigneeField.setIssue(issue);
+ assigneeField.setName(option.getAssigneeIssueField());
+ assigneeField.setType(InputSpec.USER);
+
+ User user = getUser(users, assigneeNode, logger);
+ if (user != null) {
+ assigneeField.setValue(user.getName());
+ issue.getFields().add(assigneeField);
+ } else {
+ nonExistentLogins.add(assigneeNode.get("displayName").asText());
+ }
+ }
+
+ issue.setSubmitDate(ISODateTimeFormat.dateTime()
+ .parseDateTime(fieldsNode.get("created").asText())
+ .toDate());
+
+ LastUpdate lastUpdate = new LastUpdate();
+ lastUpdate.setActivity("Opened");
+ lastUpdate.setDate(issue.getSubmitDate());
+ lastUpdate.setUser(issue.getSubmitter());
+ issue.setLastUpdate(lastUpdate);
+
+ if (option.getDueDateIssueField() != null) {
+ String dueDate = fieldsNode.get("duedate").asText(null);
+ if (dueDate != null) {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(option.getDueDateIssueField());
+ issueField.setType(InputSpec.DATE);
+ issueField.setValue(dueDate);
+ issue.getFields().add(issueField);
+ }
+ }
+
+ if (option.getTimeEstimateIssueField() != null) {
+ String timeEstimate = fieldsNode.get("timeestimate").asText(null);
+ if (timeEstimate != null) {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(option.getTimeEstimateIssueField());
+ issueField.setType(InputSpec.WORKING_PERIOD);
+ issueField.setValue(DateUtils.formatWorkingPeriod(Integer.valueOf(timeEstimate)/60));
+ issue.getFields().add(issueField);
+ }
+ }
+ if (option.getTimeSpentIssueField() != null) {
+ String timeSpent = fieldsNode.get("timespent").asText(null);
+ if (timeSpent != null) {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(option.getTimeSpentIssueField());
+ issueField.setType(InputSpec.WORKING_PERIOD);
+ issueField.setValue(DateUtils.formatWorkingPeriod(Integer.valueOf(timeSpent)/60));
+ issue.getFields().add(issueField);
+ }
+ }
+
+ if (fieldsNode.hasNonNull("subtasks")) {
+ List<String> subtasks = new ArrayList<>();
+ for (JsonNode subtaskNode: fieldsNode.get("subtasks"))
+ subtasks.add(subtaskNode.get("key").asText());
+ if (!subtasks.isEmpty())
+ extraIssueInfo.put("Sub Tasks", StringUtils.join(subtasks, "<br>"));
+ }
+
+ if (fieldsNode.hasNonNull("parent"))
+ extraIssueInfo.put("Parent Task", fieldsNode.get("parent").get("key").asText());
+
+ if (fieldsNode.hasNonNull("issuelinks")) {
+ Map<String, List<String>> issueLinks = new LinkedHashMap<>();
+ for (JsonNode issueLinkNode: fieldsNode.get("issuelinks")) {
+ if (issueLinkNode.get("inwardIssue") != null) {
+ String type = issueLinkNode.get("type").get("inward").asText();
+ List<String> linkedIssues = issueLinks.get(type);
+ if (linkedIssues == null) {
+ linkedIssues = new ArrayList<>();
+ issueLinks.put(type, linkedIssues);
+ }
+ linkedIssues.add(issueLinkNode.get("inwardIssue").get("key").asText());
+ } else {
+ String type = issueLinkNode.get("type").get("outward").asText();
+ List<String> linkedIssues = issueLinks.get(type);
+ if (linkedIssues == null) {
+ linkedIssues = new ArrayList<>();
+ issueLinks.put(type, linkedIssues);
+ }
+ linkedIssues.add(issueLinkNode.get("outwardIssue").get("key").asText());
+ }
+ }
+ for (Map.Entry<String, List<String>> entry: issueLinks.entrySet())
+ extraIssueInfo.put(entry.getKey(), StringUtils.join(entry.getValue(), "<br>"));
+ }
+
+ if (fieldsNode.hasNonNull("comment") && fieldsNode.get("comment").hasNonNull("comments")) {
+ for (JsonNode commentNode: fieldsNode.get("comment").get("comments")) {
+ String commentContent = getMarkdown(commentNode.get("body").get("content"), true);
+ if (StringUtils.isNotBlank(commentContent)) {
+ IssueComment comment = new IssueComment();
+ comment.setIssue(issue);
+ comment.setContent(commentContent);
+
+ comment.setDate(ISODateTimeFormat.dateTime()
+ .parseDateTime(commentNode.get("created").asText())
+ .toDate());
+
+ authorNode = commentNode.get("author");
+ if (authorNode != null) {
+ User user = getUser(users, authorNode, logger);
+ if (user != null) {
+ comment.setUser(user);
+ } else {
+ comment.setUser(OneDev.getInstance(UserManager.class).getUnknown());
+ nonExistentLogins.add(authorNode.get("displayName").asText());
+ }
+ } else {
+ comment.setUser(OneDev.getInstance(UserManager.class).getUnknown());
+ }
+
+ issue.getComments().add(comment);
+ }
+ }
+ }
+
+ issue.setCommentCount(issue.getComments().size());
+
+ Set<String> fieldAndValues = new HashSet<>();
+ for (IssueField field: issue.getFields()) {
+ String fieldAndValue = field.getName() + "::" + field.getValue();
+ if (!fieldAndValues.add(fieldAndValue)) {
+ String errorMessage = String.format(
+ "Duplicate issue field mapping (issue: %s, field: %s)",
+ issueKey, fieldAndValue);
+ throw new ExplicitException(errorMessage);
+ }
+ }
+
+ if (!extraIssueInfo.isEmpty()) {
+ StringBuilder builder = new StringBuilder("|");
+ for (String key: extraIssueInfo.keySet())
+ builder.append(key).append("|");
+ builder.append("\n|");
+ extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));
+ builder.append("\n|");
+ for (String value: extraIssueInfo.values())
+ builder.append(value).append("|");
+
+ if (issue.getDescription() != null)
+ issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());
+ else
+ issue.setDescription(builder.toString());
+ }
+
+ if (!dryRun) {
+ List<String> attachments = new ArrayList<>();
+
+ if (fieldsNode.hasNonNull("attachment")) {
+ long maxUploadFileSize = OneDev.getInstance(SettingManager.class)
+ .getPerformanceSetting().getMaxUploadFileSize()*1L*1024*1024;
+ for (JsonNode attachmentNode: fieldsNode.get("attachment")) {
+ String attachmentName = attachmentNode.get("filename").asText();
+ int attachmentSize = attachmentNode.get("size").asInt();
+ if (attachmentSize > maxUploadFileSize) {
+ tooLargeAttachments.add(issueKey + ":" + attachmentName);
+ } else {
+ String endpoint = attachmentNode.get("content").asText();
+ WebTarget target = client.target(endpoint);
+ Invocation.Builder builder = target.request();
+ try (Response response = builder.get()) {
+ String errorMessage = JerseyUtils.checkStatus(endpoint, response);
+ if (errorMessage != null) {
+ throw new ExplicitException(String.format(
+ "Error downloading attachment (url: %s, error message: %s)",
+ endpoint, errorMessage));
+ }
+ try (InputStream is = response.readEntity(InputStream.class)) {
+ String oneDevAttachmentName = oneDevProject.saveAttachment(issue.getUUID(), attachmentName, is);
+ String oneDevAttachmentUrl = oneDevProject.getAttachmentUrlPath(issue.getUUID(), oneDevAttachmentName);
+ attachments.add("[" + oneDevAttachmentName + "](" + oneDevAttachmentUrl + ")");
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ }
+ }
+
+ if (!attachments.isEmpty()) {
+ description = issue.getDescription();
+ if (StringUtils.isNotBlank(description))
+ description += "\n";
+ description += "#### Attachments\n" + Joiner.on(" &nbsp;&nbsp;&nbsp;&bull;&nbsp;&nbsp;&nbsp; ").join(attachments);
+ issue.setDescription(description);
+ }
+ }
+
+ issues.add(issue);
+ }
+ logger.log("Imported " + numOfImportedIssues.addAndGet(pageData.size()) + " issues");
+ }
+
+ };
+
+ String jiraProjectName = jiraProject.get("name").asText();
+ String jql = String.format("project = \"%s\" order by created DESC", jiraProjectName);
+ jql = URLEncoder.encode(jql, StandardCharsets.UTF_8.name());
+ apiEndpoint = getApiEndpoint("/search?jql=" + jql);
+ list(client, apiEndpoint, pageDataConsumer, logger);
+
+ if (!dryRun) {
+ ReferenceMigrator migrator = new ReferenceMigrator(Issue.class, issueNumberMappings);
+ String jiraProjectKey = jiraProject.get("key").asText();
+ Dao dao = OneDev.getInstance(Dao.class);
+ for (Issue issue: issues) {
+ if (issue.getDescription() != null)
+ issue.setDescription(migrator.migratePrefixed(issue.getDescription(), jiraProjectKey + "-"));
+
+ OneDev.getInstance(IssueManager.class).save(issue);
+ for (IssueSchedule schedule: issue.getSchedules())
+ dao.persist(schedule);
+ for (IssueField field: issue.getFields())
+ dao.persist(field);
+ for (IssueComment comment: issue.getComments()) {
+ comment.setContent(migrator.migratePrefixed(comment.getContent(), jiraProjectKey + "-"));
+ dao.persist(comment);
+ }
+ }
+ }
+
+ ImportResult result = new ImportResult();
+ result.unmappedIssueStatuses.addAll(unmappedIssueStatuses);
+ result.unmappedIssueTypes.addAll(unmappedIssueTypes);
+ result.unmappedIssuePriorities.addAll(unmappedIssuePriorities);
+ result.nonExistentLogins.addAll(nonExistentLogins);
+ result.tooLargeAttachments.addAll(tooLargeAttachments);
+ result.errorAttachments.addAll(errorAttachments);
+
+ return result;
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ } finally {
+ client.close();
+ }
+ }
+
+ ImportOption buildImportOption(Collection<String> projectNames) {
+ ImportOption option = new ImportOption();
+
+ TaskLogger taskLogger = new TaskLogger() {
+
+ @Override
+ public void log(String message, String sessionId) {
+ logger.info(message);
+ }
+
+ };
+ Map<String, JsonNode> projectNodes = getProjectNodes(taskLogger);
+
+ Client client = newClient();
+ try {
+ String apiEndpoint = getApiEndpoint("/status");
+ List<JsonNode> statusNodes = list(client, apiEndpoint, taskLogger);
+
+ Set<String> issueTypes = new LinkedHashSet<>();
+ Set<String> issueStatuses = new LinkedHashSet<>();
+ for (String projectName: projectNames) {
+ JsonNode projectNode = projectNodes.get(projectName);
+ if (projectNode == null)
+ throw new ExplicitException("Unable to find project: " + projectName);
+ apiEndpoint = getApiEndpoint("/issuetype/project?projectId=" + projectNode.get("id").asText());
+ for (JsonNode issueTypeNode: list(client, apiEndpoint, taskLogger))
+ issueTypes.add(issueTypeNode.get("untranslatedName").asText());
+ if (projectNode.get("style").asText().equals("next-gen")) {
+ for (JsonNode statusNode: statusNodes) {
+ JsonNode scopeNode = statusNode.get("scope");
+ if (scopeNode != null && scopeNode.get("project").get("id").asText().equals(projectNode.get("id").asText()))
+ issueStatuses.add(statusNode.get("untranslatedName").asText());
+ }
+ } else {
+ for (JsonNode statusNode: statusNodes) {
+ if (statusNode.get("scope") == null)
+ issueStatuses.add(statusNode.get("untranslatedName").asText());
+ }
+ }
+ }
+ for (String issueType: issueTypes) {
+ IssueTypeMapping mapping = new IssueTypeMapping();
+ mapping.setJiraIssueType(issueType);
+ option.getIssueTypeMappings().add(mapping);
+ }
+ for (String issueStatus: issueStatuses) {
+ IssueStatusMapping mapping = new IssueStatusMapping();
+ mapping.setJiraIssueStatus(issueStatus);
+ option.getIssueStatusMappings().add(mapping);
+ }
+ apiEndpoint = getApiEndpoint("/priority");
+ for (JsonNode priorityNode: list(client, apiEndpoint, taskLogger)) {
+ IssuePriorityMapping mapping = new IssuePriorityMapping();
+ mapping.setJiraIssuePriority(priorityNode.get("name").asText());
+ option.getIssuePriorityMappings().add(mapping);
+ }
+ } finally {
+ client.close();
+ }
+ return option;
+ }
+
+ private Map<String, JsonNode> getProjectNodes(TaskLogger logger) {
+ Map<String, JsonNode> projectNodes = new LinkedHashMap<>();
+
+ Client client = newClient();
+ try {
+ for (JsonNode projectNode: list(client, getApiEndpoint("/project/search"), logger))
+ projectNodes.put(projectNode.get("name").asText(), projectNode);
+ } finally {
+ client.close();
+ }
+ return projectNodes;
+ }
+
+ private List<JsonNode> list(Client client, String apiEndpoint, TaskLogger logger) {
+ List<JsonNode> result = new ArrayList<>();
+ list(client, apiEndpoint, new PageDataConsumer() {
+
+ @Override
+ public void consume(List<JsonNode> pageData) {
+ result.addAll(pageData);
+ }
+
+ }, logger);
+ return result;
+ }
+
+ private void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer,
+ TaskLogger logger) {
+ URI uri;
+ try {
+ uri = new URIBuilder(apiEndpoint)
+ .addParameter("maxResults", String.valueOf(PER_PAGE)).build();
+ } catch (URISyntaxException e) {
+ throw new RuntimeException(e);
+ }
+
+ int page = 0;
+ while (true) {
+ try {
+ URIBuilder builder = new URIBuilder(uri);
+ builder.addParameter("startAt", String.valueOf(page*PER_PAGE));
+ List<JsonNode> pageData = new ArrayList<>();
+
+ JsonNode resultNode = JerseyUtils.get(client, builder.build().toString(), logger);
+ if (resultNode.isArray()) {
+ for (JsonNode each: resultNode)
+ pageData.add(each);
+ } else {
+ if (resultNode.hasNonNull("values")) {
+ for (JsonNode each: resultNode.get("values"))
+ pageData.add(each);
+ } else if (resultNode.hasNonNull("issues")) {
+ for (JsonNode each: resultNode.get("issues"))
+ pageData.add(each);
+ } else if (resultNode.hasNonNull("comments")) {
+ for (JsonNode each: resultNode.get("comments"))
+ pageData.add(each);
+ }
+ }
+ pageDataConsumer.consume(pageData);
+ if (pageData.size() < PER_PAGE)
+ break;
+ page++;
+ } catch (URISyntaxException | InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
@Override
public boolean isValid(ConstraintValidatorContext context) {
Client client = ClientBuilder.newClient();
@@ -122,11 +1032,4 @@ public class ImportServer implements Serializable, Validatable {
return true;
}
- Client newClient() {
- Client client = ClientBuilder.newClient();
- client.property(ClientProperties.FOLLOW_REDIRECTS, true);
- client.register(HttpAuthenticationFeature.basic(getAccountEmail(), getApiToken()));
- return client;
- }
-
}
diff --git a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportUtils.java b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportUtils.java
deleted file mode 100644
index e920517806..0000000000
--- a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/ImportUtils.java
+++ /dev/null
@@ -1,817 +0,0 @@
-package io.onedev.server.plugin.imports.jiracloud;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.UnsupportedEncodingException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URLEncoder;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import javax.annotation.Nullable;
-import javax.ws.rs.client.Client;
-import javax.ws.rs.client.Invocation;
-import javax.ws.rs.client.WebTarget;
-import javax.ws.rs.core.Response;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.http.client.utils.URIBuilder;
-import org.joda.time.format.ISODateTimeFormat;
-import org.unbescape.html.HtmlEscape;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.google.common.base.Joiner;
-import com.google.common.base.Splitter;
-
-import io.onedev.commons.utils.ExplicitException;
-import io.onedev.commons.utils.TaskLogger;
-import io.onedev.server.OneDev;
-import io.onedev.server.entitymanager.IssueManager;
-import io.onedev.server.entitymanager.SettingManager;
-import io.onedev.server.entitymanager.UserManager;
-import io.onedev.server.entityreference.ReferenceMigrator;
-import io.onedev.server.model.Issue;
-import io.onedev.server.model.IssueComment;
-import io.onedev.server.model.IssueField;
-import io.onedev.server.model.IssueSchedule;
-import io.onedev.server.model.Project;
-import io.onedev.server.model.User;
-import io.onedev.server.model.support.LastUpdate;
-import io.onedev.server.model.support.administration.GlobalIssueSetting;
-import io.onedev.server.model.support.inputspec.InputSpec;
-import io.onedev.server.model.support.issue.field.spec.FieldSpec;
-import io.onedev.server.persistence.dao.Dao;
-import io.onedev.server.util.DateUtils;
-import io.onedev.server.util.JerseyUtils;
-import io.onedev.server.util.JerseyUtils.PageDataConsumer;
-import io.onedev.server.util.Pair;
-
-public class ImportUtils {
-
- static final String NAME = "JIRA Cloud";
-
- static final int PER_PAGE = 50;
-
- static GlobalIssueSetting getIssueSetting() {
- return OneDev.getInstance(SettingManager.class).getIssueSetting();
- }
-
- @Nullable
- static User getUser(Map<String, Optional<User>> users, JsonNode userNode, TaskLogger logger) {
- String accountId = userNode.get("accountId").asText();
- Optional<User> userOpt = users.get(accountId);
- if (userOpt == null) {
- String displayName = null;
- if (userNode.hasNonNull("displayName"))
- displayName = userNode.get("displayName").asText(null);
- if (displayName != null)
- userOpt = Optional.ofNullable(OneDev.getInstance(UserManager.class).findByFullName(displayName));
- else
- userOpt = Optional.empty();
- users.put(accountId, userOpt);
- }
- return userOpt.orElse(null);
- }
-
- private static String getMarkdown(JsonNode contentNode, boolean escape) {
- StringBuilder markdown = new StringBuilder();
- for (JsonNode itemNode: contentNode) {
- String type = itemNode.get("type").asText();
- switch (type) {
- case "text":
- String text = itemNode.get("text").asText();
- if (escape)
- text = HtmlEscape.escapeHtml5(text);
-
- JsonNode marksNode = itemNode.get("marks");
- if (marksNode != null) {
- for (JsonNode markNode: marksNode) {
- String textType = markNode.get("type").asText(null);
- switch (textType) {
- case "strong":
- text = "**" + text.replace("*", "\\*") + "**";
- break;
- case "em":
- text = "_" + text.replace("_", "\\_") + "_";
- break;
- case "code":
- text = "`" + text + "`";
- break;
- case "textColor":
- text = String.format("<span style='color:%s'>%s</span>",
- markNode.get("attrs").get("color").asText(), text);
- break;
- case "link":
- text = String.format("[%s](%s)", text, markNode.get("attrs").get("href").asText());
- break;
- }
- }
- }
- markdown.append(text);
- break;
- case "mention":
- markdown.append("@" + itemNode.get("attrs").get("text").asText());
- break;
- case "heading":
- markdown.append(StringUtils.repeat('#', itemNode.get("attrs").get("level").asInt())).append(" ");
- markdown.append(getMarkdown(itemNode.get("content"), true)).append("\n\n");
- break;
- case "paragraph":
- markdown.append(getMarkdown(itemNode.get("content"), true)).append("\n\n");
- break;
- case "bulletList":
- for (JsonNode listItemNode: itemNode.get("content"))
- markdown.append("- ").append(getMarkdown(listItemNode.get("content"), true));
- break;
- case "orderedList":
- for (JsonNode listItemNode: itemNode.get("content"))
- markdown.append("1. ").append(getMarkdown(listItemNode.get("content"), true));
- break;
- case "inlineCard":
- markdown.append(" " + StringUtils.substringAfterLast(itemNode.get("attrs").get("url").asText(), "/") + " ");
- break;
- case "emoji":
- markdown.append(" " + itemNode.get("attrs").get("shortName").asText() + " ");
- break;
- case "codeBlock":
- markdown.append("```\n" + getMarkdown(itemNode.get("content"), false) + "\n```\n\n");
- break;
- case "panel":
- String panelType = itemNode.get("attrs").get("panelType").asText();
- switch (panelType) {
- case "info":
- case "note":
- markdown.append(":information_source:\n");
- break;
- case "success":
- markdown.append(":white_check_mark:\n");
- break;
- case "warning":
- markdown.append(":warning:\n");
- break;
- case "error":
- markdown.append(":no_entry:\n");
- break;
- }
- markdown.append(getMarkdown(itemNode.get("content"), true));
- break;
- case "blockquote":
- List<String> lines = Splitter.on('\n').splitToList(getMarkdown(itemNode.get("content"), true));
- for (String line: lines)
- markdown.append("> ").append(line).append("\n");
- break;
- case "rule":
- markdown.append("---\n");
- break;
- case "date":
- markdown.append(DateUtils.formatDate(new Date(itemNode.get("attrs").get("timestamp").asLong())));
- break;
- case "status":
- JsonNode attrsNode = itemNode.get("attrs");
- text = HtmlEscape.escapeHtml5(attrsNode.get("text").asText());
- String color = attrsNode.get("color").asText();
- switch (color) {
- case "yellow":
- markdown.append(String.format("<span style='background:#FFF0B3; padding:2px 4px; border-radius:3px; font-weight:bold; color:#172B4D;'>%s</span>", text));
- break;
- case "neutral":
- markdown.append(String.format("<span style='background:#DFE1E6; padding:2px 4px; border-radius:3px; font-weight:bold; color:#42526E;'>%s</span>", text));
- break;
- case "purple":
- markdown.append(String.format("<span style='background:#EAE6FF; padding:2px 4px; border-radius:3px; font-weight:bold; color:#574BA3;'>%s</span>", text));
- break;
- case "blue":
- markdown.append(String.format("<span style='background:#DEEBFF; padding:2px 4px; border-radius:3px; font-weight:bold; color:#0747A6;'>%s</span>", text));
- break;
- case "red":
- markdown.append(String.format("<span style='background:#FFEBE6; padding:2px 4px; border-radius:3px; font-weight:bold; color:#BF2600;'>%s</span>", text));
- break;
- case "green":
- markdown.append(String.format("<span style='background:#E3FCEF; padding:2px 4px; border-radius:3px; font-weight:bold; color:#006644;'>%s</span>", text));
- break;
- default:
- markdown.append(text);
- }
- break;
- case "table":
- List<List<String>> tableData = new ArrayList<>();
- for (JsonNode rowNode: itemNode.get("content")) {
- List<String> columns = new ArrayList<>();
- for (JsonNode columnNode: rowNode.get("content"))
- columns.add(getMarkdown(columnNode.get("content"), true).replace("\n", "<br>"));
- tableData.add(columns);
- }
- if (!tableData.isEmpty()) {
- if (!tableData.get(0).isEmpty()) {
- markdown.append("|");
- for (int i=0; i<tableData.get(0).size(); i++)
- markdown.append("-----|");
- markdown.append("\n");
- for (List<String> row: tableData) {
- markdown.append("|");
- for (String column: row)
- markdown.append(" ").append(column).append(" |");
- markdown.append("\n");
- }
- markdown.append("\n");
- }
- }
- break;
- }
- }
- return markdown.toString();
- }
-
- static ImportResult importIssues(ImportServer server, JsonNode jiraProject, Project oneDevProject,
- boolean retainIssueNumbers, ImportOption importOption,
- Map<String, Optional<User>> users, boolean dryRun, TaskLogger logger) {
- Client client = server.newClient();
- try {
- Set<String> unmappedIssueStatuses = new HashSet<>();
- Set<String> unmappedIssueTypes = new HashSet<>();
- Set<String> unmappedIssuePriorities = new HashSet<>();
- Set<String> nonExistentLogins = new HashSet<>();
- Set<String> tooLargeAttachments = new LinkedHashSet<>();
- Set<String> errorAttachments = new HashSet<>();
-
- Map<String, Pair<FieldSpec, String>> typeMappings = new HashMap<>();
-
- for (IssueTypeMapping mapping: importOption.getIssueTypeMappings()) {
- String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");
- String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");
- FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);
- if (fieldSpec == null)
- throw new ExplicitException("No field spec found: " + oneDevFieldName);
- typeMappings.put(mapping.getJiraIssueType(), new Pair<>(fieldSpec, oneDevFieldValue));
- }
-
- Map<String, Pair<FieldSpec, String>> priorityMappings = new HashMap<>();
-
- for (IssuePriorityMapping mapping: importOption.getIssuePriorityMappings()) {
- String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");
- String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");
- FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);
- if (fieldSpec == null)
- throw new ExplicitException("No field spec found: " + oneDevFieldName);
- priorityMappings.put(mapping.getJiraIssuePriority(), new Pair<>(fieldSpec, oneDevFieldValue));
- }
-
- String initialIssueState = getIssueSetting().getInitialStateSpec().getName();
-
- List<Issue> issues = new ArrayList<>();
-
- Map<Long, Long> issueNumberMappings = new HashMap<>();
-
- Map<String, String> statusMappings = new HashMap<>();
- for (IssueStatusMapping mapping: importOption.getIssueStatusMappings())
- statusMappings.put(mapping.getJiraIssueStatus(), mapping.getOneDevIssueState());
-
- Map<String, String> untranslatedStatusNames = new HashMap<>();
- String apiEndpoint = server.getApiEndpoint("/status");
- for (JsonNode statusNode: list(client, apiEndpoint, logger)) {
- String untranslated = statusNode.get("untranslatedName").asText();
- untranslatedStatusNames.put(statusNode.get("id").asText(), untranslated);
- }
-
- Map<String, String> untranslatedTypeNames = new HashMap<>();
- apiEndpoint = server.getApiEndpoint("/issuetype/project?projectId=" + jiraProject.get("id").asText());
- for (JsonNode typeNode: list(client, apiEndpoint, logger))
- untranslatedTypeNames.put(typeNode.get("id").asText(), typeNode.get("untranslatedName").asText());
-
- AtomicInteger numOfImportedIssues = new AtomicInteger(0);
- PageDataConsumer pageDataConsumer = new PageDataConsumer() {
-
- private String joinAsMultilineHtml(List<String> values) {
- List<String> escapedValues = new ArrayList<>();
- for (String value: values)
- escapedValues.add(HtmlEscape.escapeHtml5(value));
- return StringUtils.join(escapedValues, "<br>");
- }
-
- @Override
- public void consume(List<JsonNode> pageData) throws InterruptedException {
- for (JsonNode issueNode: pageData) {
- if (Thread.interrupted())
- throw new InterruptedException();
-
- String issueKey = issueNode.get("key").asText();
- String apiEndpoint = server.getApiEndpoint("/issue/" + issueKey);
- issueNode = JerseyUtils.get(client, apiEndpoint, logger);
-
- Map<String, String> extraIssueInfo = new LinkedHashMap<>();
-
- Issue issue = new Issue();
- issue.setProject(oneDevProject);
-
- issue.setNumberScope(oneDevProject.getForkRoot());
-
- Long oldNumber = Long.valueOf(StringUtils.substringAfterLast(issueNode.get("key").asText(), "-"));
- Long newNumber;
- if (dryRun || retainIssueNumbers)
- newNumber = oldNumber;
- else
- newNumber = OneDev.getInstance(IssueManager.class).getNextNumber(oneDevProject);
- issue.setNumber(newNumber);
- issueNumberMappings.put(oldNumber, newNumber);
-
- JsonNode fieldsNode = issueNode.get("fields");
-
- issue.setTitle(fieldsNode.get("summary").asText());
-
- String description = "";
- JsonNode descriptionNode = fieldsNode.get("description");
- if (descriptionNode != null && descriptionNode.hasNonNull("content")) {
- description = getMarkdown(descriptionNode.get("content"), true);
- if (StringUtils.isNotBlank("description"))
- description = "#### Description\n\n" + description;
- }
-
- JsonNode environmentNode = fieldsNode.get("environment");
- if (environmentNode != null && environmentNode.hasNonNull("content")) {
- String environment = getMarkdown(environmentNode.get("content"), true);
- if (StringUtils.isNotBlank(environment)) {
- if (StringUtils.isNotBlank(description))
- description += "\n";
- description += "#### Environment\n\n" + environment;
- }
- }
-
- if (StringUtils.isNotBlank(description))
- issue.setDescription(description);
-
- String statusId = fieldsNode.get("status").get("id").asText();
- String untranslatedStatusName = untranslatedStatusNames.get(statusId);
- if (untranslatedStatusName != null) {
- String statusName = statusMappings.get(untranslatedStatusName);
- if (statusName != null) {
- issue.setState(statusName);
- } else {
- unmappedIssueStatuses.add(untranslatedStatusName);
- issue.setState(initialIssueState);
- }
- } else {
- throw new ExplicitException("Can not find JIRA status of id: " + statusId);
- }
-
- String typeId = fieldsNode.get("issuetype").get("id").asText();
- String untranslatedTypeName = untranslatedTypeNames.get(typeId);
- if (untranslatedTypeName != null) {
- Pair<FieldSpec, String> mapped = typeMappings.get(untranslatedTypeName);
- if (mapped != null) {
- IssueField typeField = new IssueField();
- typeField.setIssue(issue);
- typeField.setName(mapped.getFirst().getName());
- typeField.setType(InputSpec.ENUMERATION);
- typeField.setValue(mapped.getSecond());
- typeField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));
- issue.getFields().add(typeField);
- } else {
- extraIssueInfo.put("Type", HtmlEscape.escapeHtml5(untranslatedTypeName));
- unmappedIssueTypes.add(untranslatedTypeName);
- }
- } else {
- String errorMessage = String.format("Can not find JIRA issue type by id (project: %s, issue type id: %s)",
- jiraProject.get("name").asText(), typeId);
- throw new ExplicitException(errorMessage);
- }
-
- String priorityName = fieldsNode.get("priority").get("name").asText();
- Pair<FieldSpec, String> mapped = priorityMappings.get(priorityName);
- if (mapped != null) {
- IssueField priorityField = new IssueField();
- priorityField.setIssue(issue);
- priorityField.setName(mapped.getFirst().getName());
- priorityField.setType(InputSpec.ENUMERATION);
- priorityField.setValue(mapped.getSecond());
- priorityField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));
- issue.getFields().add(priorityField);
- } else {
- extraIssueInfo.put("Priority", HtmlEscape.escapeHtml5(priorityName));
- unmappedIssuePriorities.add(priorityName);
- }
-
- List<String> components = new ArrayList<>();
- for (JsonNode componentNode: fieldsNode.get("components"))
- components.add(componentNode.get("name").asText());
- if (!components.isEmpty())
- extraIssueInfo.put("Components", joinAsMultilineHtml(components));
-
- List<String> affectedVersions = new ArrayList<>();
- for (JsonNode versionNode: fieldsNode.get("versions"))
- affectedVersions.add(versionNode.get("name").asText());
- if (!affectedVersions.isEmpty())
- extraIssueInfo.put("Affected Versions", joinAsMultilineHtml(affectedVersions));
-
- List<String> fixVersions = new ArrayList<>();
- for (JsonNode versionNode: fieldsNode.get("fixVersions"))
- fixVersions.add(versionNode.get("name").asText());
- if (!fixVersions.isEmpty())
- extraIssueInfo.put("Fix Versions", joinAsMultilineHtml(fixVersions));
-
- if (fieldsNode.hasNonNull("labels")) {
- List<String> labels = new ArrayList<>();
- for (JsonNode labelNode: fieldsNode.get("labels"))
- labels.add(labelNode.asText());
- if (!labels.isEmpty())
- extraIssueInfo.put("Labels", joinAsMultilineHtml(labels));
- }
-
- JsonNode authorNode = fieldsNode.get("reporter");
- if (authorNode == null)
- authorNode = fieldsNode.get("creator");
- if (authorNode != null) {
- User user = getUser(users, authorNode, logger);
- if (user != null) {
- issue.setSubmitter(user);
- } else {
- issue.setSubmitter(OneDev.getInstance(UserManager.class).getUnknown());
- nonExistentLogins.add(authorNode.get("displayName").asText());
- }
- } else {
- issue.setSubmitter(OneDev.getInstance(UserManager.class).getUnknown());
- }
-
- if (fieldsNode.hasNonNull("assignee")) {
- JsonNode assigneeNode = fieldsNode.get("assignee");
- IssueField assigneeField = new IssueField();
- assigneeField.setIssue(issue);
- assigneeField.setName(importOption.getAssigneeIssueField());
- assigneeField.setType(InputSpec.USER);
-
- User user = getUser(users, assigneeNode, logger);
- if (user != null) {
- assigneeField.setValue(user.getName());
- issue.getFields().add(assigneeField);
- } else {
- nonExistentLogins.add(assigneeNode.get("displayName").asText());
- }
- }
-
- issue.setSubmitDate(ISODateTimeFormat.dateTime()
- .parseDateTime(fieldsNode.get("created").asText())
- .toDate());
-
- LastUpdate lastUpdate = new LastUpdate();
- lastUpdate.setActivity("Opened");
- lastUpdate.setDate(issue.getSubmitDate());
- lastUpdate.setUser(issue.getSubmitter());
- issue.setLastUpdate(lastUpdate);
-
- if (importOption.getDueDateIssueField() != null) {
- String dueDate = fieldsNode.get("duedate").asText(null);
- if (dueDate != null) {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(importOption.getDueDateIssueField());
- issueField.setType(InputSpec.DATE);
- issueField.setValue(dueDate);
- issue.getFields().add(issueField);
- }
- }
-
- if (importOption.getTimeEstimateIssueField() != null) {
- String timeEstimate = fieldsNode.get("timeestimate").asText(null);
- if (timeEstimate != null) {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(importOption.getTimeEstimateIssueField());
- issueField.setType(InputSpec.WORKING_PERIOD);
- issueField.setValue(DateUtils.formatWorkingPeriod(Integer.valueOf(timeEstimate)/60));
- issue.getFields().add(issueField);
- }
- }
- if (importOption.getTimeSpentIssueField() != null) {
- String timeSpent = fieldsNode.get("timespent").asText(null);
- if (timeSpent != null) {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(importOption.getTimeSpentIssueField());
- issueField.setType(InputSpec.WORKING_PERIOD);
- issueField.setValue(DateUtils.formatWorkingPeriod(Integer.valueOf(timeSpent)/60));
- issue.getFields().add(issueField);
- }
- }
-
- if (fieldsNode.hasNonNull("subtasks")) {
- List<String> subtasks = new ArrayList<>();
- for (JsonNode subtaskNode: fieldsNode.get("subtasks"))
- subtasks.add(subtaskNode.get("key").asText());
- if (!subtasks.isEmpty())
- extraIssueInfo.put("Sub Tasks", StringUtils.join(subtasks, "<br>"));
- }
-
- if (fieldsNode.hasNonNull("parent"))
- extraIssueInfo.put("Parent Task", fieldsNode.get("parent").get("key").asText());
-
- if (fieldsNode.hasNonNull("issuelinks")) {
- Map<String, List<String>> issueLinks = new LinkedHashMap<>();
- for (JsonNode issueLinkNode: fieldsNode.get("issuelinks")) {
- if (issueLinkNode.get("inwardIssue") != null) {
- String type = issueLinkNode.get("type").get("inward").asText();
- List<String> linkedIssues = issueLinks.get(type);
- if (linkedIssues == null) {
- linkedIssues = new ArrayList<>();
- issueLinks.put(type, linkedIssues);
- }
- linkedIssues.add(issueLinkNode.get("inwardIssue").get("key").asText());
- } else {
- String type = issueLinkNode.get("type").get("outward").asText();
- List<String> linkedIssues = issueLinks.get(type);
- if (linkedIssues == null) {
- linkedIssues = new ArrayList<>();
- issueLinks.put(type, linkedIssues);
- }
- linkedIssues.add(issueLinkNode.get("outwardIssue").get("key").asText());
- }
- }
- for (Map.Entry<String, List<String>> entry: issueLinks.entrySet())
- extraIssueInfo.put(entry.getKey(), StringUtils.join(entry.getValue(), "<br>"));
- }
-
- if (fieldsNode.hasNonNull("comment") && fieldsNode.get("comment").hasNonNull("comments")) {
- for (JsonNode commentNode: fieldsNode.get("comment").get("comments")) {
- String commentContent = getMarkdown(commentNode.get("body").get("content"), true);
- if (StringUtils.isNotBlank(commentContent)) {
- IssueComment comment = new IssueComment();
- comment.setIssue(issue);
- comment.setContent(commentContent);
-
- comment.setDate(ISODateTimeFormat.dateTime()
- .parseDateTime(commentNode.get("created").asText())
- .toDate());
-
- authorNode = commentNode.get("author");
- if (authorNode != null) {
- User user = getUser(users, authorNode, logger);
- if (user != null) {
- comment.setUser(user);
- } else {
- comment.setUser(OneDev.getInstance(UserManager.class).getUnknown());
- nonExistentLogins.add(authorNode.get("displayName").asText());
- }
- } else {
- comment.setUser(OneDev.getInstance(UserManager.class).getUnknown());
- }
-
- issue.getComments().add(comment);
- }
- }
- }
-
- issue.setCommentCount(issue.getComments().size());
-
- Set<String> fieldAndValues = new HashSet<>();
- for (IssueField field: issue.getFields()) {
- String fieldAndValue = field.getName() + "::" + field.getValue();
- if (!fieldAndValues.add(fieldAndValue)) {
- String errorMessage = String.format(
- "Duplicate issue field mapping (issue: %s, field: %s)",
- issueKey, fieldAndValue);
- throw new ExplicitException(errorMessage);
- }
- }
-
- if (!extraIssueInfo.isEmpty()) {
- StringBuilder builder = new StringBuilder("|");
- for (String key: extraIssueInfo.keySet())
- builder.append(key).append("|");
- builder.append("\n|");
- extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));
- builder.append("\n|");
- for (String value: extraIssueInfo.values())
- builder.append(value).append("|");
-
- if (issue.getDescription() != null)
- issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());
- else
- issue.setDescription(builder.toString());
- }
-
- if (!dryRun) {
- List<String> attachments = new ArrayList<>();
-
- if (fieldsNode.hasNonNull("attachment")) {
- long maxUploadFileSize = OneDev.getInstance(SettingManager.class)
- .getPerformanceSetting().getMaxUploadFileSize()*1L*1024*1024;
- for (JsonNode attachmentNode: fieldsNode.get("attachment")) {
- String attachmentName = attachmentNode.get("filename").asText();
- int attachmentSize = attachmentNode.get("size").asInt();
- if (attachmentSize > maxUploadFileSize) {
- tooLargeAttachments.add(issueKey + ":" + attachmentName);
- } else {
- String endpoint = attachmentNode.get("content").asText();
- WebTarget target = client.target(endpoint);
- Invocation.Builder builder = target.request();
- try (Response response = builder.get()) {
- String errorMessage = JerseyUtils.checkStatus(endpoint, response);
- if (errorMessage != null) {
- throw new ExplicitException(String.format(
- "Error downloading attachment (url: %s, error message: %s)",
- endpoint, errorMessage));
- }
- try (InputStream is = response.readEntity(InputStream.class)) {
- String oneDevAttachmentName = oneDevProject.saveAttachment(issue.getUUID(), attachmentName, is);
- String oneDevAttachmentUrl = oneDevProject.getAttachmentUrlPath(issue.getUUID(), oneDevAttachmentName);
- attachments.add("[" + oneDevAttachmentName + "](" + oneDevAttachmentUrl + ")");
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
- }
- }
-
- if (!attachments.isEmpty()) {
- description = issue.getDescription();
- if (StringUtils.isNotBlank(description))
- description += "\n";
- description += "#### Attachments\n" + Joiner.on(" &nbsp;&nbsp;&nbsp;&bull;&nbsp;&nbsp;&nbsp; ").join(attachments);
- issue.setDescription(description);
- }
- }
-
- issues.add(issue);
- }
- logger.log("Imported " + numOfImportedIssues.addAndGet(pageData.size()) + " issues");
- }
-
- };
-
- String jiraProjectName = jiraProject.get("name").asText();
- String jql = String.format("project = \"%s\" order by created DESC", jiraProjectName);
- jql = URLEncoder.encode(jql, StandardCharsets.UTF_8.name());
- apiEndpoint = server.getApiEndpoint("/search?jql=" + jql);
- list(client, apiEndpoint, pageDataConsumer, logger);
-
- if (!dryRun) {
- ReferenceMigrator migrator = new ReferenceMigrator(Issue.class, issueNumberMappings);
- String jiraProjectKey = jiraProject.get("key").asText();
- Dao dao = OneDev.getInstance(Dao.class);
- for (Issue issue: issues) {
- if (issue.getDescription() != null)
- issue.setDescription(migrator.migratePrefixed(issue.getDescription(), jiraProjectKey + "-"));
-
- OneDev.getInstance(IssueManager.class).save(issue);
- for (IssueSchedule schedule: issue.getSchedules())
- dao.persist(schedule);
- for (IssueField field: issue.getFields())
- dao.persist(field);
- for (IssueComment comment: issue.getComments()) {
- comment.setContent(migrator.migratePrefixed(comment.getContent(), jiraProjectKey + "-"));
- dao.persist(comment);
- }
- }
- }
-
- ImportResult result = new ImportResult();
- result.unmappedIssueStatuses.addAll(unmappedIssueStatuses);
- result.unmappedIssueTypes.addAll(unmappedIssueTypes);
- result.unmappedIssuePriorities.addAll(unmappedIssuePriorities);
- result.nonExistentLogins.addAll(nonExistentLogins);
- result.tooLargeAttachments.addAll(tooLargeAttachments);
- result.errorAttachments.addAll(errorAttachments);
-
- return result;
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e);
- } finally {
- client.close();
- }
- }
-
- static ImportOption buildImportOption(ImportServer server, Collection<String> projectNames, TaskLogger logger) {
- ImportOption importOption = new ImportOption();
-
- Map<String, JsonNode> projectNodes = getProjectNodes(server, logger);
-
- Client client = server.newClient();
- try {
- String apiEndpoint = server.getApiEndpoint("/status");
- List<JsonNode> statusNodes = list(client, apiEndpoint, logger);
-
- Set<String> issueTypes = new LinkedHashSet<>();
- Set<String> issueStatuses = new LinkedHashSet<>();
- for (String projectName: projectNames) {
- JsonNode projectNode = projectNodes.get(projectName);
- if (projectNode == null)
- throw new ExplicitException("Unable to find project: " + projectName);
- apiEndpoint = server.getApiEndpoint("/issuetype/project?projectId=" + projectNode.get("id").asText());
- for (JsonNode issueTypeNode: list(client, apiEndpoint, logger))
- issueTypes.add(issueTypeNode.get("untranslatedName").asText());
- if (projectNode.get("style").asText().equals("next-gen")) {
- for (JsonNode statusNode: statusNodes) {
- JsonNode scopeNode = statusNode.get("scope");
- if (scopeNode != null && scopeNode.get("project").get("id").asText().equals(projectNode.get("id").asText()))
- issueStatuses.add(statusNode.get("untranslatedName").asText());
- }
- } else {
- for (JsonNode statusNode: statusNodes) {
- if (statusNode.get("scope") == null)
- issueStatuses.add(statusNode.get("untranslatedName").asText());
- }
- }
- }
- for (String issueType: issueTypes) {
- IssueTypeMapping mapping = new IssueTypeMapping();
- mapping.setJiraIssueType(issueType);
- importOption.getIssueTypeMappings().add(mapping);
- }
- for (String issueStatus: issueStatuses) {
- IssueStatusMapping mapping = new IssueStatusMapping();
- mapping.setJiraIssueStatus(issueStatus);
- importOption.getIssueStatusMappings().add(mapping);
- }
- apiEndpoint = server.getApiEndpoint("/priority");
- for (JsonNode priorityNode: list(client, apiEndpoint, logger)) {
- IssuePriorityMapping mapping = new IssuePriorityMapping();
- mapping.setJiraIssuePriority(priorityNode.get("name").asText());
- importOption.getIssuePriorityMappings().add(mapping);
- }
- } finally {
- client.close();
- }
- return importOption;
- }
-
- static Map<String, JsonNode> getProjectNodes(ImportServer where, TaskLogger logger) {
- Map<String, JsonNode> projectNodes = new LinkedHashMap<>();
-
- Client client = where.newClient();
- try {
- for (JsonNode projectNode: list(client, where.getApiEndpoint("/project/search"), logger))
- projectNodes.put(projectNode.get("name").asText(), projectNode);
- } finally {
- client.close();
- }
- return projectNodes;
- }
-
- static List<JsonNode> list(Client client, String apiEndpoint, TaskLogger logger) {
- List<JsonNode> result = new ArrayList<>();
- list(client, apiEndpoint, new PageDataConsumer() {
-
- @Override
- public void consume(List<JsonNode> pageData) {
- result.addAll(pageData);
- }
-
- }, logger);
- return result;
- }
-
- static void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer,
- TaskLogger logger) {
- URI uri;
- try {
- uri = new URIBuilder(apiEndpoint)
- .addParameter("maxResults", String.valueOf(PER_PAGE)).build();
- } catch (URISyntaxException e) {
- throw new RuntimeException(e);
- }
-
- int page = 0;
- while (true) {
- try {
- URIBuilder builder = new URIBuilder(uri);
- builder.addParameter("startAt", String.valueOf(page*PER_PAGE));
- List<JsonNode> pageData = new ArrayList<>();
-
- JsonNode resultNode = JerseyUtils.get(client, builder.build().toString(), logger);
- if (resultNode.isArray()) {
- for (JsonNode each: resultNode)
- pageData.add(each);
- } else {
- if (resultNode.hasNonNull("values")) {
- for (JsonNode each: resultNode.get("values"))
- pageData.add(each);
- } else if (resultNode.hasNonNull("issues")) {
- for (JsonNode each: resultNode.get("issues"))
- pageData.add(each);
- } else if (resultNode.hasNonNull("comments")) {
- for (JsonNode each: resultNode.get("comments"))
- pageData.add(each);
- }
- }
- pageDataConsumer.consume(pageData);
- if (pageData.size() < PER_PAGE)
- break;
- page++;
- } catch (URISyntaxException | InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
-}
diff --git a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/IssueImportSource.java b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/IssueImportSource.java
deleted file mode 100644
index f1cff0927d..0000000000
--- a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/IssueImportSource.java
+++ /dev/null
@@ -1,68 +0,0 @@
-package io.onedev.server.plugin.imports.jiracloud;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import javax.ws.rs.client.Client;
-
-import org.hibernate.validator.constraints.NotEmpty;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.databind.JsonNode;
-
-import io.onedev.commons.utils.TaskLogger;
-import io.onedev.server.web.editable.annotation.ChoiceProvider;
-import io.onedev.server.web.editable.annotation.Editable;
-import io.onedev.server.web.util.WicketUtils;
-
-@Editable
-public class IssueImportSource implements Serializable {
-
- private static final long serialVersionUID = 1L;
-
- private static final Logger logger = LoggerFactory.getLogger(IssueImportSource.class);
-
- private String project;
-
- @Editable(order=400, name="JIRA Project", description="Choose JIRA project to import issues from")
- @ChoiceProvider("getProjectChoices")
- @NotEmpty
- public String getProject() {
- return project;
- }
-
- public void setProject(String project) {
- this.project = project;
- }
-
- @SuppressWarnings("unused")
- private static List<String> getProjectChoices() {
- List<String> choices = new ArrayList<>();
-
- ImportServer server = WicketUtils.getPage().getMetaData(ImportServer.META_DATA_KEY);
-
- Client client = server.newClient();
- try {
- String apiEndpoint = server.getApiEndpoint("/project/search");
- for (JsonNode projectNode: ImportUtils.list(client, apiEndpoint, new TaskLogger() {
-
- @Override
- public void log(String message, String sessionId) {
- logger.info(message);
- }
-
- })) {
- choices.add(projectNode.get("name").asText());
- }
- } finally {
- client.close();
- }
-
- Collections.sort(choices);
- return choices;
- }
-
-}
diff --git a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/JiraIssueImporter.java b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/JiraIssueImporter.java
index b0855cd526..164f8c03f5 100644
--- a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/JiraIssueImporter.java
+++ b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/JiraIssueImporter.java
@@ -1,58 +1,86 @@
package io.onedev.server.plugin.imports.jiracloud;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
+import java.io.Serializable;
+import java.util.List;
-import javax.ws.rs.client.Client;
-
-import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.collect.Lists;
-import io.onedev.commons.utils.ExplicitException;
import io.onedev.commons.utils.TaskLogger;
import io.onedev.server.imports.IssueImporter;
import io.onedev.server.model.Project;
-import io.onedev.server.model.User;
-import io.onedev.server.web.util.WicketUtils;
+import io.onedev.server.web.util.ImportStep;
-public class JiraIssueImporter extends IssueImporter<ImportServer, IssueImportSource, ImportOption> {
+public class JiraIssueImporter implements IssueImporter {
private static final long serialVersionUID = 1L;
- @Override
- public String getName() {
- return ImportUtils.NAME;
- }
+ private final ImportStep<ImportServer> serverStep = new ImportStep<ImportServer>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Authenticate to JIRA cloud";
+ }
+
+ @Override
+ protected ImportServer newSetting() {
+ return new ImportServer();
+ }
+
+ };
+
+ private final ImportStep<ImportProject> projectStep = new ImportStep<ImportProject>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Choose project";
+ }
+
+ @Override
+ protected ImportProject newSetting() {
+ ImportProject project = new ImportProject();
+ project.server = serverStep.getSetting();
+ return project;
+ }
+
+ };
+
+ private final ImportStep<ImportOption> optionStep = new ImportStep<ImportOption>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Specify import option";
+ }
+ @Override
+ protected ImportOption newSetting() {
+ return serverStep.getSetting().buildImportOption(
+ Lists.newArrayList(projectStep.getSetting().getProject()));
+ }
+
+ };
+
@Override
- public IssueImportSource getWhat(ImportServer where, TaskLogger logger) {
- WicketUtils.getPage().setMetaData(ImportServer.META_DATA_KEY, where);
- return new IssueImportSource();
+ public String getName() {
+ return JiraPluginModule.NAME;
}
@Override
- public ImportOption getHow(ImportServer where, IssueImportSource what, TaskLogger logger) {
- return ImportUtils.buildImportOption(where, Lists.newArrayList(what.getProject()), logger);
+ public String doImport(Project project, boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {
+ ImportServer server = serverStep.getSetting();
+ String jiraProject = projectStep.getSetting().getProject();
+ ImportOption option = optionStep.getSetting();
+ return server.importIssues(project, jiraProject, option, retainIssueNumbers, dryRun, logger);
}
@Override
- public String doImport(ImportServer where, IssueImportSource what, ImportOption how,
- Project project, boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {
- Map<String, JsonNode> projectNodes = ImportUtils.getProjectNodes(where, logger);
-
- Client client = where.newClient();
- try {
- Map<String, Optional<User>> users = new HashMap<>();
- JsonNode projectNode = projectNodes.get(what.getProject());
- if (projectNode == null)
- throw new ExplicitException("Unable to find project: " + what.getProject());
- ImportResult result = ImportUtils.importIssues(where, projectNode, project, retainIssueNumbers,
- how, users, dryRun, logger);
- return result.toHtml("Issues imported successfully");
- } finally {
- client.close();
- }
+ public List<ImportStep<? extends Serializable>> getSteps() {
+ return Lists.newArrayList(serverStep, projectStep, optionStep);
}
}
diff --git a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/JiraPluginModule.java b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/JiraPluginModule.java
index 386e3c8540..d2dbe40475 100644
--- a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/JiraPluginModule.java
+++ b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/JiraPluginModule.java
@@ -1,6 +1,5 @@
package io.onedev.server.plugin.imports.jiracloud;
-import java.io.Serializable;
import java.util.Collection;
import com.google.common.collect.Lists;
@@ -17,6 +16,8 @@ import io.onedev.server.imports.ProjectImporterContribution;
*/
public class JiraPluginModule extends AbstractPluginModule {
+ static final String NAME = "JIRA Cloud";
+
@Override
protected void configure() {
super.configure();
@@ -25,7 +26,7 @@ public class JiraPluginModule extends AbstractPluginModule {
contribute(ProjectImporterContribution.class, new ProjectImporterContribution() {
@Override
- public Collection<ProjectImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> getImporters() {
+ public Collection<ProjectImporter> getImporters() {
return Lists.newArrayList(new JiraProjectImporter());
}
@@ -39,7 +40,7 @@ public class JiraPluginModule extends AbstractPluginModule {
contribute(IssueImporterContribution.class, new IssueImporterContribution() {
@Override
- public Collection<IssueImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> getImporters() {
+ public Collection<IssueImporter> getImporters() {
return Lists.newArrayList(new JiraIssueImporter());
}
diff --git a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/JiraProjectImporter.java b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/JiraProjectImporter.java
index a79e5532fc..baf4c3e42e 100644
--- a/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/JiraProjectImporter.java
+++ b/server-plugin/server-plugin-import-jiracloud/src/main/java/io/onedev/server/plugin/imports/jiracloud/JiraProjectImporter.java
@@ -1,109 +1,90 @@
package io.onedev.server.plugin.imports.jiracloud;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
+import java.io.Serializable;
import java.util.List;
-import java.util.Map;
-import java.util.Optional;
import java.util.stream.Collectors;
-import javax.ws.rs.client.Client;
+import com.google.common.collect.Lists;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.google.common.base.Preconditions;
-
-import io.onedev.commons.utils.ExplicitException;
import io.onedev.commons.utils.TaskLogger;
-import io.onedev.server.OneDev;
-import io.onedev.server.entitymanager.ProjectManager;
import io.onedev.server.imports.ProjectImporter;
-import io.onedev.server.model.Project;
-import io.onedev.server.model.User;
-import io.onedev.server.storage.StorageManager;
-import io.onedev.server.util.JerseyUtils;
+import io.onedev.server.web.util.ImportStep;
-public class JiraProjectImporter extends ProjectImporter<ImportServer, ProjectImportSource, ImportOption> {
+public class JiraProjectImporter implements ProjectImporter {
private static final long serialVersionUID = 1L;
+
+ private final ImportStep<ImportServer> serverStep = new ImportStep<ImportServer>() {
- @Override
- public String getName() {
- return ImportUtils.NAME;
- }
+ private static final long serialVersionUID = 1L;
- @Override
- public ProjectImportSource getWhat(ImportServer where, TaskLogger logger) {
- ProjectImportSource importSource = new ProjectImportSource();
- for (String projectName: ImportUtils.getProjectNodes(where, logger).keySet()) {
- ProjectMapping mapping = new ProjectMapping();
- mapping.setJiraProject(projectName);
- mapping.setOneDevProject(projectName.replace(' ', '-'));
- importSource.getProjectMappings().add(mapping);
+ @Override
+ public String getTitle() {
+ return "Authenticate to JIRA cloud";
+ }
+
+ @Override
+ protected ImportServer newSetting() {
+ return new ImportServer();
+ }
+
+ };
+
+ private final ImportStep<ImportProjects> projectsStep = new ImportStep<ImportProjects>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Specify projects";
+ }
+
+ @Override
+ protected ImportProjects newSetting() {
+ ImportProjects projects = new ImportProjects();
+ for (String project: serverStep.getSetting().listProjects()) {
+ ProjectMapping mapping = new ProjectMapping();
+ mapping.setJiraProject(project);
+ mapping.setOneDevProject(project.replace(' ', '-'));
+ projects.getProjectMappings().add(mapping);
+ }
+ return projects;
+ }
+
+ };
+
+ private final ImportStep<ImportOption> optionStep = new ImportStep<ImportOption>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Specify import option";
+ }
+
+ @Override
+ protected ImportOption newSetting() {
+ List<String> jiraProjects = projectsStep.getSetting().getProjectMappings().stream()
+ .map(it->it.getJiraProject()).collect(Collectors.toList());
+ return serverStep.getSetting().buildImportOption(jiraProjects);
}
- return importSource;
+
+ };
+
+ @Override
+ public String getName() {
+ return JiraPluginModule.NAME;
}
@Override
- public ImportOption getHow(ImportServer where, ProjectImportSource what, TaskLogger logger) {
- List<String> jiraProjects = what.getProjectMappings().stream()
- .map(it->it.getJiraProject()).collect(Collectors.toList());
- return ImportUtils.buildImportOption(where, jiraProjects, logger);
+ public String doImport(boolean dryRun, TaskLogger logger) {
+ return serverStep.getSetting().importProjects(
+ projectsStep.getSetting(), optionStep.getSetting(), dryRun, logger);
}
@Override
- public String doImport(ImportServer where, ProjectImportSource what, ImportOption how,
- boolean dryRun, TaskLogger logger) {
- Collection<Long> projectIds = new ArrayList<>();
-
- Map<String, JsonNode> projectNodes = ImportUtils.getProjectNodes(where, logger);
-
- Client client = where.newClient();
- try {
- Map<String, Optional<User>> users = new HashMap<>();
- ImportResult result = new ImportResult();
-
- for (ProjectMapping projectMapping: what.getProjectMappings()) {
- String jiraProject = projectMapping.getJiraProject();
- JsonNode projectNode = projectNodes.get(jiraProject);
- if (projectNode == null)
- throw new ExplicitException("Unable to find project: " + jiraProject);
-
- String apiEndpoint = where.getApiEndpoint("/project/" + projectNode.get("id").asText());
-
- // Get more detail project information
- projectNode = JerseyUtils.get(client, apiEndpoint, logger);
-
- ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
- Project project = projectManager.initialize(projectMapping.getOneDevProject());
- Preconditions.checkState(project.isNew());
-
- project.setDescription(projectNode.get("description").asText(null));
-
- if (!dryRun) {
- OneDev.getInstance(ProjectManager.class).create(project);
- projectIds.add(project.getId());
- }
-
- logger.log("Importing issues from project " + jiraProject + "...");
- ImportResult currentResult = ImportUtils.importIssues(where, projectNode,
- project, true, how, users, dryRun, logger);
- result.nonExistentLogins.addAll(currentResult.nonExistentLogins);
- result.errorAttachments.addAll(currentResult.errorAttachments);
- result.tooLargeAttachments.addAll(currentResult.tooLargeAttachments);
- result.unmappedIssuePriorities.addAll(currentResult.unmappedIssuePriorities);
- result.unmappedIssueStatuses.addAll(currentResult.unmappedIssueStatuses);
- result.unmappedIssueTypes.addAll(currentResult.unmappedIssueTypes);
- }
-
- return result.toHtml("Projects imported successfully");
- } catch (Exception e) {
- for (Long projectId: projectIds)
- OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);
- throw new RuntimeException(e);
- } finally {
- client.close();
- }
+ public List<ImportStep<? extends Serializable>> getSteps() {
+ return Lists.newArrayList(serverStep, projectsStep, optionStep);
}
}
diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/IssueImportSource.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportProject.java
similarity index 56%
rename from server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/IssueImportSource.java
rename to server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportProject.java
index bff3d7a5cb..f4cf37f4b7 100644
--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/IssueImportSource.java
+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportProject.java
@@ -5,20 +5,21 @@ import java.util.List;
import org.hibernate.validator.constraints.NotEmpty;
+import io.onedev.server.util.ComponentContext;
+import io.onedev.server.web.editable.BeanEditor;
import io.onedev.server.web.editable.annotation.ChoiceProvider;
import io.onedev.server.web.editable.annotation.Editable;
-import io.onedev.server.web.util.WicketUtils;
@Editable
-public class IssueImportSource implements Serializable {
+public class ImportProject implements Serializable {
private static final long serialVersionUID = 1L;
+ ImportServer server;
+
private String project;
- @Editable(order=100, name="YouTrack Project", description="Choose YouTrack project to "
- + "import issues from. Available projects will be populated when above options "
- + "are specified")
+ @Editable(order=400, name="YouTrack Project", description="Choose YouTrack project to import issues from")
@ChoiceProvider("getProjectChoices")
@NotEmpty
public String getProject() {
@@ -31,7 +32,9 @@ public class IssueImportSource implements Serializable {
@SuppressWarnings("unused")
private static List<String> getProjectChoices() {
- return WicketUtils.getPage().getMetaData(ImportServer.META_DATA_KEY).getProjectChoices();
+ BeanEditor editor = ComponentContext.get().getComponent().findParent(BeanEditor.class);
+ ImportProject setting = (ImportProject) editor.getModelObject();
+ return setting.server.listProjects();
}
}
diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ProjectImportSource.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportProjects.java
similarity index 93%
rename from server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ProjectImportSource.java
rename to server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportProjects.java
index c399f9d421..8a4635e834 100644
--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ProjectImportSource.java
+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportProjects.java
@@ -15,16 +15,20 @@ import io.onedev.server.model.Project;
import io.onedev.server.util.validation.Validatable;
import io.onedev.server.util.validation.annotation.ClassValidating;
import io.onedev.server.web.editable.annotation.Editable;
+import io.onedev.server.web.editable.annotation.OmitName;
@Editable
@ClassValidating
-public class ProjectImportSource implements Serializable, Validatable {
+public class ImportProjects implements Serializable, Validatable {
private static final long serialVersionUID = 1L;
+ ImportServer server;
+
private List<ProjectMapping> projectMappings = new ArrayList<>();
@Editable(order=100, name="Projects to Import")
+ @OmitName
@Size(min=1, max=10000, message="No projects to import")
public List<ProjectMapping> getProjectMappings() {
return projectMappings;
diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportServer.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportServer.java
index 04bc86fa68..1fe86b0701 100644
--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportServer.java
+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportServer.java
@@ -1,10 +1,25 @@
package io.onedev.server.plugin.imports.youtrack;
+import java.io.IOException;
+import java.io.InputStream;
import java.io.Serializable;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
import javax.validation.ConstraintValidatorContext;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
@@ -12,17 +27,58 @@ import javax.ws.rs.client.Invocation;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response;
+import org.apache.http.client.utils.URIBuilder;
import org.apache.wicket.MetaDataKey;
import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
import org.hibernate.validator.constraints.NotEmpty;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.unbescape.html.HtmlEscape;
import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
+import io.onedev.commons.utils.ExceptionUtils;
+import io.onedev.commons.utils.ExplicitException;
import io.onedev.commons.utils.StringUtils;
import io.onedev.commons.utils.TaskLogger;
+import io.onedev.server.OneDev;
+import io.onedev.server.entitymanager.IssueLinkManager;
+import io.onedev.server.entitymanager.IssueManager;
+import io.onedev.server.entitymanager.LinkSpecManager;
+import io.onedev.server.entitymanager.ProjectManager;
+import io.onedev.server.entitymanager.SettingManager;
+import io.onedev.server.entitymanager.UserManager;
+import io.onedev.server.entityreference.ReferenceMigrator;
+import io.onedev.server.model.Issue;
+import io.onedev.server.model.IssueComment;
+import io.onedev.server.model.IssueField;
+import io.onedev.server.model.IssueLink;
+import io.onedev.server.model.IssueSchedule;
+import io.onedev.server.model.LinkSpec;
+import io.onedev.server.model.Project;
+import io.onedev.server.model.User;
+import io.onedev.server.model.support.LastUpdate;
+import io.onedev.server.model.support.administration.GlobalIssueSetting;
+import io.onedev.server.model.support.inputspec.InputSpec;
+import io.onedev.server.model.support.issue.StateSpec;
+import io.onedev.server.model.support.issue.field.spec.ChoiceField;
+import io.onedev.server.model.support.issue.field.spec.DateField;
+import io.onedev.server.model.support.issue.field.spec.DateTimeField;
+import io.onedev.server.model.support.issue.field.spec.FieldSpec;
+import io.onedev.server.model.support.issue.field.spec.FloatField;
+import io.onedev.server.model.support.issue.field.spec.IntegerField;
+import io.onedev.server.model.support.issue.field.spec.TextField;
+import io.onedev.server.model.support.issue.field.spec.UserChoiceField;
+import io.onedev.server.model.support.issue.field.spec.WorkingPeriodField;
+import io.onedev.server.persistence.dao.Dao;
+import io.onedev.server.security.SecurityUtils;
+import io.onedev.server.storage.StorageManager;
+import io.onedev.server.util.DateUtils;
import io.onedev.server.util.JerseyUtils;
+import io.onedev.server.util.Pair;
+import io.onedev.server.util.JerseyUtils.PageDataConsumer;
import io.onedev.server.util.validation.Validatable;
import io.onedev.server.util.validation.annotation.ClassValidating;
import io.onedev.server.web.editable.annotation.Editable;
@@ -36,6 +92,8 @@ public class ImportServer implements Serializable, Validatable {
private static final Logger logger = LoggerFactory.getLogger(ImportServer.class);
+ private static final int PER_PAGE = 50;
+
static final MetaDataKey<ImportServer> META_DATA_KEY = new MetaDataKey<ImportServer>() {
private static final long serialVersionUID = 1L;
@@ -90,6 +148,1027 @@ public class ImportServer implements Serializable, Validatable {
return StringUtils.stripEnd(apiUrl, "/") + "/" + StringUtils.stripStart(apiPath, "/");
}
+ List<String> listProjects() {
+ List<String> projects = new ArrayList<>();
+
+ Client client = newClient();
+ try {
+ TaskLogger logger = new TaskLogger() {
+
+ @Override
+ public void log(String message, String sessionId) {
+ ImportServer.logger.info(message);
+ }
+
+ };
+ String apiEndpoint = getApiEndpoint("/admin/projects?fields=name");
+ for (JsonNode projectNode: list(client, apiEndpoint, logger))
+ projects.add(projectNode.get("name").asText());
+ } finally {
+ client.close();
+ }
+
+ Collections.sort(projects);
+ return projects;
+ }
+
+ private Client newClient() {
+ Client client = ClientBuilder.newClient();
+ client.register(HttpAuthenticationFeature.basic(getUserName(), getPassword()));
+ return client;
+ }
+
+ ImportOption buildImportOption(ImportProjects projects) {
+ Client client = newClient();
+ try {
+ List<JsonNode> projectNodes = new ArrayList<>();
+ String apiEndpoint = getApiEndpoint("/admin/projects?fields=id,name,customFields(field(name),bundle(values(name)))");
+ Set<String> projectNames = projects.getProjectMappings().stream()
+ .map(it->it.getYouTrackProject()).collect(Collectors.toSet());
+ for (JsonNode projectNode: list(client, apiEndpoint, new TaskLogger() {
+
+ @Override
+ public void log(String message, String sessionId) {
+ logger.info(message);
+ }
+
+ })) {
+ if (projectNames.contains(projectNode.get("name").asText()))
+ projectNodes.add(projectNode);
+ }
+
+ return buildImportOption(projectNodes);
+ } finally {
+ client.close();
+ }
+ }
+
+ ImportOption buildImportOption(String project) {
+ Client client = newClient();
+ try {
+ String apiEndpoint = getApiEndpoint("/admin/projects?fields=id,name,customFields(field(name),bundle(values(name)))");
+ for (JsonNode projectNode: list(client, apiEndpoint, new TaskLogger() {
+
+ @Override
+ public void log(String message, String sessionId) {
+ logger.info(message);
+ }
+
+ })) {
+ if (project.equals(projectNode.get("name").asText())) {
+ List<JsonNode> projectNodes = new ArrayList<>();
+ projectNodes.add(projectNode);
+ return buildImportOption(projectNodes);
+ }
+ }
+ throw new ExplicitException("Unable to find YouTrack project: " + project);
+ } finally {
+ client.close();
+ }
+ }
+
+ ImportOption buildImportOption(Collection<JsonNode> projectNodes) {
+ ImportOption option = new ImportOption();
+ Client client = newClient();
+ try {
+ Set<String> youTrackIssueStates = new LinkedHashSet<>();
+ Set<String> youTrackIssueFields = new LinkedHashSet<>();
+
+ for (JsonNode projectNode: projectNodes) {
+ for (JsonNode customFieldNode: projectNode.get("customFields")) {
+ String fieldName = customFieldNode.get("field").get("name").asText();
+ JsonNode bundleNode = customFieldNode.get("bundle");
+ if (bundleNode != null) {
+ String bundleType = bundleNode.get("$type").asText();
+ if (bundleType.equals("StateBundle")) {
+ for (JsonNode valueNode: bundleNode.get("values"))
+ youTrackIssueStates.add(valueNode.get("name").asText());
+ } else if (bundleType.equals("EnumBundle") || bundleType.equals("OwnedBundle")) {
+ for (JsonNode valueNode: bundleNode.get("values"))
+ youTrackIssueFields.add(fieldName + "::" + valueNode.get("name").asText());
+ } else {
+ youTrackIssueFields.add(fieldName);
+ }
+ } else {
+ youTrackIssueFields.add(fieldName);
+ }
+ }
+ }
+
+ for (String youTrackIssueState: youTrackIssueStates) {
+ IssueStateMapping mapping = new IssueStateMapping();
+ mapping.setYouTrackIssueState(youTrackIssueState);
+ option.getIssueStateMappings().add(mapping);
+ }
+
+ for (String youTrackIssueField: youTrackIssueFields) {
+ IssueFieldMapping mapping = new IssueFieldMapping();
+ mapping.setYouTrackIssueField(youTrackIssueField);
+ option.getIssueFieldMappings().add(mapping);
+ }
+
+ TaskLogger taskLogger = new TaskLogger() {
+
+ @Override
+ public void log(String message, String sessionId) {
+ logger.info(message);
+ }
+
+ };
+ String apiEndpoint = getApiEndpoint("/issueTags?fields=name");
+ for (JsonNode tagNode: list(client, apiEndpoint, taskLogger)) {
+ IssueTagMapping mapping = new IssueTagMapping();
+ mapping.setYouTrackIssueTag(tagNode.get("name").asText());
+ option.getIssueTagMappings().add(mapping);
+ }
+
+ apiEndpoint = getApiEndpoint("/issueLinkTypes?fields=name");
+ for (JsonNode tagNode: list(client, apiEndpoint, taskLogger)) {
+ IssueLinkMapping mapping = new IssueLinkMapping();
+ mapping.setYouTrackIssueLink(tagNode.get("name").asText());
+ option.getIssueLinkMappings().add(mapping);
+ }
+
+ } finally {
+ client.close();
+ }
+ return option;
+ }
+
+ private ImportResult importIssues(String youTrackProjectId, Project oneDevProject, boolean retainIssueNumbers,
+ ImportOption option, boolean dryRun, TaskLogger logger) {
+ Client client = newClient();
+ try {
+ String apiEndpoint = getApiEndpoint("/admin/projects/" + youTrackProjectId + "?fields=shortName");
+ String youTrackProjectShortName = JerseyUtils.get(client, apiEndpoint, logger).get("shortName").asText();
+
+ Map<String, String> stateMappings = new HashMap<>();
+ Map<String, Pair<FieldSpec, String>> fieldMappings = new HashMap<>();
+ Map<String, Pair<FieldSpec, String>> tagMappings = new HashMap<>();
+ Map<String, LinkSpec> linkMappings = new HashMap<>();
+
+ Map<Long, Long> issueNumberMappings = new HashMap<>();
+ Map<Long, Issue> issueMappings = new HashMap<>();
+
+ for (IssueStateMapping mapping: option.getIssueStateMappings())
+ stateMappings.put(mapping.getYouTrackIssueState(), mapping.getOneDevIssueState());
+
+ for (IssueFieldMapping mapping: option.getIssueFieldMappings()) {
+ String oneDevFieldName;
+ String oneDevFieldValue;
+ if (mapping.getOneDevIssueField().contains("::")) {
+ oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");
+ oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");
+ } else {
+ oneDevFieldName = mapping.getOneDevIssueField();
+ oneDevFieldValue = null;
+ }
+ FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);
+ if (fieldSpec == null)
+ throw new ExplicitException("No field spec found: " + oneDevFieldName);
+ fieldMappings.put(mapping.getYouTrackIssueField(), new Pair<>(fieldSpec, oneDevFieldValue));
+ }
+ for (IssueTagMapping mapping: option.getIssueTagMappings()) {
+ String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");
+ String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");
+ FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);
+ if (fieldSpec == null)
+ throw new ExplicitException("No field spec found: " + oneDevFieldName);
+ tagMappings.put(mapping.getYouTrackIssueTag(), new Pair<>(fieldSpec, oneDevFieldValue));
+ }
+ for (IssueLinkMapping mapping: option.getIssueLinkMappings()) {
+ LinkSpec linkSpec = OneDev.getInstance(LinkSpecManager.class).find(mapping.getOneDevIssueLink());
+ if (linkSpec == null)
+ throw new ExplicitException("No link spec found: " + mapping.getOneDevIssueLink());
+ linkMappings.put(mapping.getYouTrackIssueLink(), linkSpec);
+ }
+
+ Set<String> nonExistentLogins = new LinkedHashSet<>();
+ Set<String> unmappedIssueTags = new LinkedHashSet<>();
+ Set<String> unmappedIssueFields = new LinkedHashSet<>();
+ Set<String> unmappedIssueLinks = new LinkedHashSet<>();
+ Set<String> unmappedIssueStates = new LinkedHashSet<>();
+ Map<String, String> mismatchedIssueFields = new LinkedHashMap<>();
+ Set<String> tooLargeAttachments = new LinkedHashSet<>();
+
+ AtomicInteger numOfImportedIssues = new AtomicInteger(0);
+
+ List<Issue> issues = new ArrayList<>();
+ Map<Long, Pair<LinkSpec, List<Long>>> issueLinkInfo = new HashMap<>();
+ Map<Long, Set<Set<Long>>> processedSymmetricLinks = new HashMap<>();
+
+ String fields = ""
+ + "idReadable,"
+ + "numberInProject,"
+ + "summary,"
+ + "description,"
+ + "created,"
+ + "comments(created,text,author(login,name,email),attachments(name,url,size)),"
+ + "attachments(name,url,comment,size),"
+ + "reporter(login,name,email),"
+ + "tags(name),"
+ + "customFields(name,value(name,login,email,presentation,text),projectCustomField(field(fieldType(id)))),"
+ + "links(direction,linkType(name,sourceToTarget,targetToSource),issues(numberInProject))";
+ PageDataConsumer pageDataConsumer = new PageDataConsumer() {
+
+ @Nullable
+ private String processAttachments(String issueUUID, String readableIssueId, @Nullable String markdown,
+ List<JsonNode> attachmentNodes, Set<String> tooLargeAttachments) {
+ if (markdown == null)
+ markdown = "";
+
+ Map<String, String> unreferencedAttachments = new LinkedHashMap<>();
+
+ long maxUploadFileSize = OneDev.getInstance(SettingManager.class)
+ .getPerformanceSetting().getMaxUploadFileSize()*1L*1024*1024;
+ for (JsonNode attachmentNode: attachmentNodes) {
+ String attachmentName = attachmentNode.get("name").asText(null);
+ String attachmentUrl = attachmentNode.get("url").asText(null);
+ long attachmentSize = attachmentNode.get("size").asLong(0);
+ if (attachmentSize != 0 && attachmentName != null && attachmentUrl != null) {
+ if (attachmentSize > maxUploadFileSize) {
+ tooLargeAttachments.add(readableIssueId + ":" + attachmentName);
+ } else {
+ if (!attachmentUrl.startsWith("/api"))
+ throw new ExplicitException("Unexpected attachment url: " + attachmentUrl);
+ attachmentUrl = attachmentUrl.substring("/api".length());
+
+ String endpoint = getApiEndpoint(attachmentUrl);
+ WebTarget target = client.target(endpoint);
+ Invocation.Builder builder = target.request();
+ try (Response response = builder.get()) {
+ String errorMessage = JerseyUtils.checkStatus(endpoint, response);
+ if (errorMessage != null) {
+ throw new ExplicitException(String.format(
+ "Error downloading attachment (url: %s, error message: %s)",
+ endpoint, errorMessage));
+ }
+ try (InputStream is = response.readEntity(InputStream.class)) {
+ String oneDevAttachmentName = oneDevProject.saveAttachment(issueUUID, attachmentName, is);
+ String oneDevAttachmentUrl = oneDevProject.getAttachmentUrlPath(issueUUID, oneDevAttachmentName);
+ if (markdown.contains("(" + attachmentName + ")")) {
+ markdown = markdown.replace("(" + attachmentName + ")", "(" + oneDevAttachmentUrl + ")");
+ } else {
+ unreferencedAttachments.put(attachmentName, oneDevAttachmentUrl);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ }
+ }
+
+ if (!unreferencedAttachments.isEmpty()) {
+ markdown += "\n\n";
+ for (Map.Entry<String, String> entry: unreferencedAttachments.entrySet())
+ markdown += "[" + entry.getKey() + "](" + entry.getValue() + ") ";
+ }
+
+ if (markdown.length() == 0)
+ markdown = null;
+
+ return markdown;
+ }
+
+ private String joinAsMultilineHtml(List<String> values) {
+ List<String> escapedValues = new ArrayList<>();
+ for (String value: values)
+ escapedValues.add(HtmlEscape.escapeHtml5(value));
+ return StringUtils.join(escapedValues, "<br>");
+ }
+
+ @Nullable
+ private String getEmail(JsonNode userNode) {
+ JsonNode emailNode = userNode.get("email");
+ if (emailNode != null)
+ return emailNode.asText(null);
+ else
+ return null;
+ }
+
+ @Override
+ public void consume(List<JsonNode> pageData) throws InterruptedException {
+ for (JsonNode issueNode: pageData) {
+ if (Thread.interrupted())
+ throw new InterruptedException();
+
+ Issue issue = new Issue();
+ String readableId = issueNode.get("idReadable").asText();
+ Long oldNumber = issueNode.get("numberInProject").asLong();
+ Long newNumber;
+ if (dryRun || retainIssueNumbers)
+ newNumber = oldNumber;
+ else
+ newNumber = getIssueManager().getNextNumber(oneDevProject);
+ issue.setNumber(newNumber);
+ issueNumberMappings.put(oldNumber, newNumber);
+ issueMappings.put(oldNumber, issue);
+ issue.setTitle(issueNode.get("summary").asText());
+ issue.setDescription(issueNode.get("description").asText(null));
+ issue.setSubmitDate(new Date(issueNode.get("created").asLong(System.currentTimeMillis())));
+ issue.setProject(oneDevProject);
+ issue.setNumberScope(oneDevProject.getForkRoot());
+
+ if (!dryRun) {
+ List<JsonNode> attachmentNodes = new ArrayList<>();
+ for (JsonNode attachmentNode: issueNode.get("attachments")) {
+ if (!attachmentNode.hasNonNull("comment"))
+ attachmentNodes.add(attachmentNode);
+ }
+ issue.setDescription(processAttachments(issue.getUUID(), readableId,
+ issue.getDescription(), attachmentNodes, tooLargeAttachments));
+ }
+
+ Map<String, String> extraIssueInfo = new LinkedHashMap<>();
+
+ if (issueNode.hasNonNull("reporter")) {
+ JsonNode reporterNode = issueNode.get("reporter");
+ String email = getEmail(reporterNode);
+ String login = reporterNode.get("login").asText();
+ if (email != null) {
+ User user = OneDev.getInstance(UserManager.class).findByVerifiedEmailAddress(email);
+ if (user != null) {
+ issue.setSubmitter(user);
+ } else {
+ issue.setSubmitter(OneDev.getInstance(UserManager.class).getUnknown());
+ nonExistentLogins.add(login);
+ }
+ } else {
+ issue.setSubmitter(OneDev.getInstance(UserManager.class).getUnknown());
+ nonExistentLogins.add(login);
+ }
+ } else {
+ issue.setSubmitter(SecurityUtils.getUser());
+ }
+
+ LastUpdate lastUpdate = new LastUpdate();
+ lastUpdate.setActivity("Opened");
+ lastUpdate.setDate(issue.getSubmitDate());
+ lastUpdate.setUser(issue.getSubmitter());
+ issue.setLastUpdate(lastUpdate);
+
+ StateSpec initialState = getIssueSetting().getInitialStateSpec();
+ for (JsonNode customFieldNode: issueNode.get("customFields")) {
+ String fieldName = customFieldNode.get("name").asText();
+ String fieldType = customFieldNode.get("$type").asText();
+ switch (fieldType) {
+ case "StateIssueCustomField":
+ if (customFieldNode.hasNonNull("value")) {
+ String state = customFieldNode.get("value").get("name").asText();
+ String mappedState = stateMappings.get(state);
+ if (mappedState != null)
+ issue.setState(mappedState);
+ else
+ unmappedIssueStates.add(state);
+ }
+ break;
+ case "SingleBuildIssueCustomField":
+ case "SingleGroupIssueCustomField":
+ case "SingleVersionIssueCustomField":
+ String fieldValue = customFieldNode.get("value").asText(null);
+ if (fieldValue != null) {
+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
+ if (mapped == null) {
+ unmappedIssueFields.add(fieldName);
+ extraIssueInfo.put(fieldName, fieldValue);
+ } else if (!(mapped.getFirst() instanceof TextField)) {
+ mismatchedIssueFields.put(fieldName, "Should be mapped to a text field");
+ extraIssueInfo.put(fieldName, fieldValue);
+ } else {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(mapped.getFirst().getName());
+ issueField.setType(InputSpec.TEXT);
+ issueField.setValue(fieldValue);
+ issue.getFields().add(issueField);
+ }
+ }
+ break;
+ case "MultiBuildIssueCustomField":
+ case "MultiGroupIssueCustomField":
+ case "MultiVersionIssueCustomField":
+ List<String> fieldValues = new ArrayList<>();
+ for (JsonNode valueNode: customFieldNode.get("value"))
+ fieldValues.add(valueNode.get("name").asText());
+ if (!fieldValues.isEmpty()) {
+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
+ if (mapped == null) {
+ unmappedIssueFields.add(fieldName);
+ extraIssueInfo.put(fieldName, joinAsMultilineHtml(fieldValues));
+ } else if (!(mapped.getFirst() instanceof TextField)) {
+ mismatchedIssueFields.put(fieldName, "Should be mapped to a text field");
+ extraIssueInfo.put(fieldName, joinAsMultilineHtml(fieldValues));
+ } else {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(mapped.getFirst().getName());
+ issueField.setType(InputSpec.TEXT);
+ issueField.setValue(StringUtils.join(fieldValues, ", "));
+ issue.getFields().add(issueField);
+ }
+ }
+ break;
+ case "SingleEnumIssueCustomField":
+ case "SingleOwnedIssueCustomField":
+ if (customFieldNode.hasNonNull("value")) {
+ fieldValue = customFieldNode.get("value").get("name").asText();
+ String fieldNameWithValue = fieldName + "::" + fieldValue;
+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldNameWithValue);
+ if (mapped == null) {
+ unmappedIssueFields.add(fieldNameWithValue);
+ extraIssueInfo.put(fieldName, fieldValue);
+ } else if (!(mapped.getFirst() instanceof ChoiceField) || mapped.getFirst().isAllowMultiple()) {
+ mismatchedIssueFields.put(fieldNameWithValue, "Should be mapped to a single-valued enum field");
+ extraIssueInfo.put(fieldName, fieldValue);
+ } else {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(mapped.getFirst().getName());
+ issueField.setType(InputSpec.ENUMERATION);
+ issueField.setValue(mapped.getSecond());
+ issue.getFields().add(issueField);
+ }
+ }
+ break;
+ case "MultiEnumIssueCustomField":
+ case "MultiOwnedIssueCustomField":
+ fieldValues = new ArrayList<>();
+ for (JsonNode valueNode: customFieldNode.get("value"))
+ fieldValues.add(valueNode.get("name").asText());
+
+ boolean hasErrors = false;
+ List<IssueField> issueFields = new ArrayList<>();
+ for (String each: fieldValues) {
+ String fieldNameWithValue = fieldName + "::" + each;
+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldNameWithValue);
+ if (mapped == null) {
+ unmappedIssueFields.add(fieldNameWithValue);
+ hasErrors = true;
+ } else if (!(mapped.getFirst() instanceof ChoiceField) || !mapped.getFirst().isAllowMultiple()) {
+ mismatchedIssueFields.put(fieldNameWithValue, "Should be mapped to a multi-valued enum field");
+ hasErrors = true;
+ } else {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(mapped.getFirst().getName());
+ issueField.setType(InputSpec.ENUMERATION);
+ issueField.setValue(mapped.getSecond());
+ issueFields.add(issueField);
+ }
+ }
+ if (hasErrors)
+ extraIssueInfo.put(fieldName, joinAsMultilineHtml(fieldValues));
+ else
+ issue.getFields().addAll(issueFields);
+ break;
+ case "SingleUserIssueCustomField":
+ if (customFieldNode.hasNonNull("value")) {
+ JsonNode valueNode = customFieldNode.get("value");
+
+ String login = valueNode.get("login").asText();
+ String fullName = valueNode.get("name").asText();
+ String email = getEmail(valueNode);
+
+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
+ if (mapped == null) {
+ unmappedIssueFields.add(fieldName);
+ extraIssueInfo.put(fieldName, fullName);
+ } else if (!(mapped.getFirst() instanceof UserChoiceField) || mapped.getFirst().isAllowMultiple()) {
+ mismatchedIssueFields.put(fieldName, "Should be mapped to a single-valued user field");
+ extraIssueInfo.put(fieldName, fullName);
+ } else {
+ if (email != null) {
+ User user = OneDev.getInstance(UserManager.class).findByVerifiedEmailAddress(email);
+ if (user != null) {
+ fieldValue = user.getName();
+ } else {
+ fieldValue = null;
+ nonExistentLogins.add(login);
+ }
+ } else {
+ fieldValue = null;
+ nonExistentLogins.add(login);
+ }
+
+ if (fieldValue != null) {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(mapped.getFirst().getName());
+ issueField.setType(InputSpec.USER);
+ issueField.setValue(fieldValue);
+ issue.getFields().add(issueField);
+ }
+ }
+ }
+ break;
+ case "MultiUserIssueCustomField":
+ List<String> fullNames = new ArrayList<>();
+ for (JsonNode valueNode: customFieldNode.get("value"))
+ fullNames.add(valueNode.get("name").asText());
+ if (!fullNames.isEmpty()) {
+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
+ if (mapped == null) {
+ unmappedIssueFields.add(fieldName);
+ extraIssueInfo.put(fieldName, joinAsMultilineHtml(fullNames));
+ } else if (!(mapped.getFirst() instanceof UserChoiceField) || !mapped.getFirst().isAllowMultiple()) {
+ mismatchedIssueFields.put(fieldName, "Should be mapped to a multi-valued user field");
+ extraIssueInfo.put(fieldName, joinAsMultilineHtml(fullNames));
+ } else {
+ for (JsonNode valueNode: customFieldNode.get("value")) {
+ String login = valueNode.get("login").asText();
+ String email = getEmail(valueNode);
+
+ if (email != null) {
+ User user = OneDev.getInstance(UserManager.class).findByVerifiedEmailAddress(email);
+ if (user != null) {
+ fieldValue = user.getName();
+ } else {
+ fieldValue = null;
+ nonExistentLogins.add(login);
+ }
+ } else {
+ fieldValue = null;
+ nonExistentLogins.add(login);
+ }
+
+ if (fieldValue != null) {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(mapped.getFirst().getName());
+ issueField.setType(InputSpec.USER);
+ issueField.setValue(fieldValue);
+ issue.getFields().add(issueField);
+ }
+ }
+ }
+ }
+ break;
+ case "DateIssueCustomField":
+ if (customFieldNode.hasNonNull("value")) {
+ Date date = new Date(customFieldNode.get("value").asLong());
+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
+ if (mapped == null) {
+ unmappedIssueFields.add(fieldName);
+ extraIssueInfo.put(fieldName, DateUtils.formatDate(date));
+ } else if (!(mapped.getFirst() instanceof DateField)) {
+ mismatchedIssueFields.put(fieldName, "Should be mapped to a date field");
+ extraIssueInfo.put(fieldName, DateUtils.formatDate(date));
+ } else {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(mapped.getFirst().getName());
+ issueField.setType(InputSpec.DATE);
+ issueField.setValue(DateUtils.formatDate(date));
+ issue.getFields().add(issueField);
+ }
+ }
+ break;
+ case "SimpleIssueCustomField":
+ if (customFieldNode.hasNonNull("value")) {
+ JsonNode projectCustomField = customFieldNode.get("projectCustomField");
+ JsonNode fieldNode = projectCustomField.get("field");
+ if (fieldNode != null) {
+ switch (fieldNode.get("fieldType").get("id").asText()) {
+ case "date":
+ Date date = new Date(customFieldNode.get("value").asLong());
+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
+ if (mapped == null) {
+ unmappedIssueFields.add(fieldName);
+ extraIssueInfo.put(fieldName, DateUtils.formatDateTime(date));
+ } else if (!(mapped.getFirst() instanceof DateTimeField)) {
+ mismatchedIssueFields.put(fieldName, "Should be mapped to a datetime field");
+ extraIssueInfo.put(fieldName, DateUtils.formatDateTime(date));
+ } else {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(mapped.getFirst().getName());
+ issueField.setType(InputSpec.DATE_TIME);
+ issueField.setValue(DateUtils.formatDateTime(date));
+ issue.getFields().add(issueField);
+ }
+ break;
+ case "float":
+ fieldValue = customFieldNode.get("value").asText();
+ mapped = fieldMappings.get(fieldName);
+ if (mapped == null) {
+ unmappedIssueFields.add(fieldName);
+ extraIssueInfo.put(fieldName, fieldValue);
+ } else if (!(mapped.getFirst() instanceof FloatField)) {
+ mismatchedIssueFields.put(fieldName, "Should be mapped to a float field");
+ extraIssueInfo.put(fieldName, fieldValue);
+ } else {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(mapped.getFirst().getName());
+ issueField.setType(InputSpec.FLOAT);
+ issueField.setValue(fieldValue);
+ issue.getFields().add(issueField);
+ }
+ break;
+ case "integer":
+ fieldValue = customFieldNode.get("value").asText();
+ mapped = fieldMappings.get(fieldName);
+ if (mapped == null) {
+ unmappedIssueFields.add(fieldName);
+ extraIssueInfo.put(fieldName, fieldValue);
+ } else if (!(mapped.getFirst() instanceof IntegerField)) {
+ mismatchedIssueFields.put(fieldName, "Should be mapped to an integer field");
+ extraIssueInfo.put(fieldName, fieldValue);
+ } else {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(mapped.getFirst().getName());
+ issueField.setType(InputSpec.INTEGER);
+ issueField.setValue(fieldValue);
+ issue.getFields().add(issueField);
+ }
+ break;
+ case "string":
+ fieldValue = customFieldNode.get("value").asText();
+ mapped = fieldMappings.get(fieldName);
+ if (mapped == null) {
+ unmappedIssueFields.add(fieldName);
+ extraIssueInfo.put(fieldName, fieldValue);
+ } else if (!(mapped.getFirst() instanceof TextField)) {
+ mismatchedIssueFields.put(fieldName, "Should be mapped to a text field");
+ extraIssueInfo.put(fieldName, fieldValue);
+ } else {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(mapped.getFirst().getName());
+ issueField.setType(InputSpec.TEXT);
+ issueField.setValue(fieldValue);
+ issue.getFields().add(issueField);
+ }
+ break;
+ }
+ }
+ }
+ break;
+ case "TextIssueCustomField":
+ if (customFieldNode.hasNonNull("value")) {
+ fieldValue = customFieldNode.get("value").get("text").asText();
+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
+ if (mapped == null) {
+ unmappedIssueFields.add(fieldName);
+ extraIssueInfo.put(fieldName, fieldValue);
+ } else if (!(mapped.getFirst() instanceof TextField)
+ || !((TextField)mapped.getFirst()).isMultiline()) {
+ mismatchedIssueFields.put(fieldName, "Should be mapped to a multi-line text field");
+ extraIssueInfo.put(fieldName, fieldValue);
+ } else {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(mapped.getFirst().getName());
+ issueField.setType(InputSpec.TEXT);
+ issueField.setValue(fieldValue);
+ issue.getFields().add(issueField);
+ }
+ }
+ break;
+ case "PeriodIssueCustomField":
+ if (customFieldNode.hasNonNull("value")) {
+ fieldValue = customFieldNode.get("value").get("presentation").asText();
+ Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
+ if (mapped == null) {
+ unmappedIssueFields.add(fieldName);
+ extraIssueInfo.put(fieldName, fieldValue);
+ } else if (!(mapped.getFirst() instanceof WorkingPeriodField)) {
+ mismatchedIssueFields.put(fieldName, "Should be mapped to a working period field");
+ extraIssueInfo.put(fieldName, fieldValue);
+ } else {
+ IssueField issueField = new IssueField();
+ issueField.setIssue(issue);
+ issueField.setName(mapped.getFirst().getName());
+ issueField.setType(InputSpec.WORKING_PERIOD);
+ issueField.setValue(fieldValue);
+ issue.getFields().add(issueField);
+ }
+ }
+ break;
+ }
+ }
+ if (issue.getState() == null)
+ issue.setState(initialState.getName());
+
+ for (JsonNode linkNode: issueNode.get("links")) {
+ List<Long> linkedIssueNumbers = new ArrayList<>();
+ for (JsonNode linkedIssueNode: linkNode.get("issues"))
+ linkedIssueNumbers.add(linkedIssueNode.get("numberInProject").asLong());
+
+ if (!linkedIssueNumbers.isEmpty() && linkNode.hasNonNull("linkType")) {
+ JsonNode linkTypeNode = linkNode.get("linkType");
+ String direction = linkNode.get("direction").asText();
+ String linkName = linkTypeNode.get("name").asText(null);
+ if (linkName != null) {
+ LinkSpec linkSpec = linkMappings.get(linkName);
+ if (linkSpec == null) {
+ List<String> linkedIssueLinks = new ArrayList<>();
+ for (Long issueNumber: linkedIssueNumbers)
+ linkedIssueLinks.add(youTrackProjectShortName + "-" + issueNumber);
+
+ unmappedIssueLinks.add(linkName);
+ switch (direction) {
+ case "BOTH":
+ case "OUTWARD":
+ linkName = linkTypeNode.get("sourceToTarget").asText(null);
+ if (linkName != null)
+ extraIssueInfo.put(linkName, joinAsMultilineHtml(linkedIssueLinks));
+ break;
+ case "INWARD":
+ linkName = linkTypeNode.get("targetToSource").asText(null);
+ if (linkName != null)
+ extraIssueInfo.put(linkName, joinAsMultilineHtml(linkedIssueLinks));
+ break;
+ }
+ } else if ("OUTWARD".equals(direction)) {
+ issueLinkInfo.put(oldNumber, new Pair<>(linkSpec, linkedIssueNumbers));
+ } else if ("BOTH".equals(direction)) {
+ Set<Set<Long>> value = processedSymmetricLinks.get(linkSpec.getId());
+ if (value == null) {
+ value = new HashSet<>();
+ processedSymmetricLinks.put(linkSpec.getId(), value);
+ }
+ List<Long> filteredIssueNumbers = new ArrayList<>();
+ for (Long issueNumber: linkedIssueNumbers) {
+ Set<Long> linkSides = Sets.newHashSet(oldNumber, issueNumber);
+ if (value.add(linkSides))
+ filteredIssueNumbers.add(issueNumber);
+ }
+ if (!filteredIssueNumbers.isEmpty())
+ issueLinkInfo.put(oldNumber, new Pair<>(linkSpec, filteredIssueNumbers));
+ }
+ }
+ }
+ }
+
+ List<String> currentUnmappedTags = new ArrayList<>();
+ for (JsonNode tagNode: issueNode.get("tags")) {
+ String tagName = tagNode.get("name").asText();
+ Pair<FieldSpec, String> mapped = tagMappings.get(tagName);
+ if (mapped != null) {
+ IssueField tagField = new IssueField();
+ tagField.setIssue(issue);
+ tagField.setName(mapped.getFirst().getName());
+ tagField.setType(InputSpec.ENUMERATION);
+ tagField.setValue(mapped.getSecond());
+ tagField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));
+ issue.getFields().add(tagField);
+ } else {
+ currentUnmappedTags.add(HtmlEscape.escapeHtml5(tagName));
+ unmappedIssueTags.add(HtmlEscape.escapeHtml5(tagName));
+ }
+ }
+ if (!currentUnmappedTags.isEmpty())
+ extraIssueInfo.put("Tags", joinAsMultilineHtml(currentUnmappedTags));
+
+ for (JsonNode commentNode: issueNode.get("comments")) {
+ String commentContent = commentNode.get("text").asText(null);
+ if (commentContent != null || !commentNode.get("attachments").isEmpty()) {
+ IssueComment comment = new IssueComment();
+ comment.setIssue(issue);
+ if (!dryRun) {
+ List<JsonNode> attachmentNodes = new ArrayList<>();
+ for (JsonNode attachmentNode: commentNode.get("attachments"))
+ attachmentNodes.add(attachmentNode);
+ comment.setContent(processAttachments(issue.getUUID(), readableId,
+ commentContent, attachmentNodes, tooLargeAttachments));
+ }
+ comment.setDate(new Date(commentNode.get("created").asLong(System.currentTimeMillis())));
+ if (commentNode.hasNonNull("author")) {
+ JsonNode authorNode = commentNode.get("author");
+ String email = getEmail(authorNode);
+ String login = authorNode.get("login").asText();
+ if (email != null) {
+ User user = OneDev.getInstance(UserManager.class).findByVerifiedEmailAddress(email);
+ if (user != null) {
+ comment.setUser(user);
+ } else {
+ comment.setUser(OneDev.getInstance(UserManager.class).getUnknown());
+ nonExistentLogins.add(login);
+ }
+ } else {
+ comment.setUser(OneDev.getInstance(UserManager.class).getUnknown());
+ nonExistentLogins.add(login);
+ }
+ } else {
+ issue.setSubmitter(SecurityUtils.getUser());
+ }
+ issue.getComments().add(comment);
+ }
+ }
+
+ issue.setCommentCount(issue.getComments().size());
+
+ Set<String> fieldAndValues = new HashSet<>();
+ for (IssueField field: issue.getFields()) {
+ String fieldAndValue = field.getName() + "::" + field.getValue();
+ if (!fieldAndValues.add(fieldAndValue)) {
+ String errorMessage = String.format(
+ "Duplicate issue field mapping (issue: %s, field: %s)",
+ readableId, fieldAndValue);
+ throw new ExplicitException(errorMessage);
+ }
+ }
+
+ if (!extraIssueInfo.isEmpty()) {
+ StringBuilder builder = new StringBuilder("|");
+ for (String key: extraIssueInfo.keySet())
+ builder.append(key).append("|");
+ builder.append("\n|");
+ extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));
+ builder.append("\n|");
+ for (String value: extraIssueInfo.values())
+ builder.append(value).append("|");
+
+ if (issue.getDescription() != null)
+ issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());
+ else
+ issue.setDescription(builder.toString());
+ }
+ issues.add(issue);
+ }
+ logger.log("Imported " + numOfImportedIssues.addAndGet(pageData.size()) + " issues");
+ }
+
+ };
+
+ apiEndpoint = getApiEndpoint("/admin/projects/" + youTrackProjectId + "/issues?fields=" + fields);
+ list(client, apiEndpoint, pageDataConsumer, logger);
+
+ if (!dryRun) {
+ ReferenceMigrator migrator = new ReferenceMigrator(Issue.class, issueNumberMappings);
+ Dao dao = OneDev.getInstance(Dao.class);
+ for (Issue issue: issues) {
+ if (issue.getDescription() != null)
+ issue.setDescription(migrator.migratePrefixed(issue.getDescription(), youTrackProjectShortName + "-"));
+
+ getIssueManager().save(issue);
+ for (IssueSchedule schedule: issue.getSchedules())
+ dao.persist(schedule);
+ for (IssueField field: issue.getFields())
+ dao.persist(field);
+ for (IssueComment comment: issue.getComments()) {
+ comment.setContent(migrator.migratePrefixed(comment.getContent(), youTrackProjectShortName + "-"));
+ dao.persist(comment);
+ }
+ }
+
+ for (Map.Entry<Long, Pair<LinkSpec, List<Long>>> entry: issueLinkInfo.entrySet()) {
+ Issue source = issueMappings.get(entry.getKey());
+ if (source != null) {
+ for (Long targetNumber: entry.getValue().getSecond()) {
+ Issue target = issueMappings.get(targetNumber);
+ if (target != null) {
+ IssueLink link = new IssueLink();
+ link.setSource(source);
+ link.setTarget(target);
+ link.setSpec(entry.getValue().getFirst());
+ OneDev.getInstance(IssueLinkManager.class).save(link);
+ }
+ }
+ }
+ }
+ }
+
+ ImportResult result = new ImportResult();
+ result.mismatchedIssueFields.putAll(mismatchedIssueFields);
+ result.nonExistentLogins.addAll(nonExistentLogins);
+ result.tooLargeAttachments.addAll(tooLargeAttachments);
+ result.unmappedIssueFields.addAll(unmappedIssueFields);
+ result.unmappedIssueStates.addAll(unmappedIssueStates);
+ result.unmappedIssueLinks.addAll(unmappedIssueLinks);
+ result.unmappedIssueTags.addAll(unmappedIssueTags);
+
+ return result;
+ } finally {
+ client.close();
+ }
+ }
+
+ private IssueManager getIssueManager() {
+ return OneDev.getInstance(IssueManager.class);
+ }
+
+ private GlobalIssueSetting getIssueSetting() {
+ return OneDev.getInstance(SettingManager.class).getIssueSetting();
+ }
+
+ private List<JsonNode> list(Client client, String apiEndpoint, TaskLogger logger) {
+ List<JsonNode> result = new ArrayList<>();
+ list(client, apiEndpoint, new PageDataConsumer() {
+
+ @Override
+ public void consume(List<JsonNode> pageData) {
+ result.addAll(pageData);
+ }
+
+ }, logger);
+ return result;
+ }
+
+ String importProjects(ImportProjects projects, ImportOption option, boolean dryRun, TaskLogger logger) {
+ Map<String, String> youTrackProjectIds = new HashMap<>();
+ Map<String, String> youTrackProjectDescriptions = new HashMap<>();
+ Collection<Long> projectIds = new ArrayList<>();
+
+ Client client = newClient();
+ try {
+ String apiEndpoint = getApiEndpoint("/admin/projects?fields=id,name,description");
+ for (JsonNode projectNode: list(client, apiEndpoint, logger)) {
+ String projectName = projectNode.get("name").asText();
+ youTrackProjectIds.put(projectName, projectNode.get("id").asText());
+ youTrackProjectDescriptions.put(projectName, projectNode.get("description").asText(null));
+ }
+
+ ImportResult result = new ImportResult();
+
+ for (ProjectMapping projectMapping: projects.getProjectMappings()) {
+ logger.log("Importing project '" + projectMapping.getYouTrackProject() + "'...");
+
+ String youTrackProjectId = youTrackProjectIds.get(projectMapping.getYouTrackProject());
+ if (youTrackProjectId == null)
+ throw new ExplicitException("Unable to find YouTrack project: " + projectMapping.getYouTrackProject());
+
+ ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
+ Project project = projectManager.initialize(projectMapping.getOneDevProject());
+ Preconditions.checkState(project.isNew());
+ project.setDescription(youTrackProjectDescriptions.get(projectMapping.getYouTrackProject()));
+ project.setIssueManagement(true);
+
+ if (!dryRun) {
+ projectManager.create(project);
+ projectIds.add(project.getId());
+ }
+
+ ImportResult currentResult = importIssues(youTrackProjectId, project,
+ true, option, dryRun, logger);
+ result.mismatchedIssueFields.putAll(currentResult.mismatchedIssueFields);
+ result.nonExistentLogins.addAll(currentResult.nonExistentLogins);
+ result.tooLargeAttachments.addAll(currentResult.tooLargeAttachments);
+ result.unmappedIssueFields.addAll(currentResult.unmappedIssueFields);
+ result.unmappedIssueStates.addAll(currentResult.unmappedIssueStates);
+ result.unmappedIssueLinks.addAll(currentResult.unmappedIssueLinks);
+ result.unmappedIssueTags.addAll(currentResult.unmappedIssueTags);
+ }
+
+ return result.toHtml("Projects imported successfully");
+ } catch (Exception e) {
+ for (Long projectId: projectIds)
+ OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);
+ throw ExceptionUtils.unchecked(e);
+ } finally {
+ client.close();
+ }
+ }
+
+ private void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer, TaskLogger logger) {
+ URI uri;
+ try {
+ uri = new URIBuilder(apiEndpoint)
+ .addParameter("$top", String.valueOf(PER_PAGE)).build();
+ } catch (URISyntaxException e) {
+ throw new RuntimeException(e);
+ }
+
+ int page = 0;
+ while (true) {
+ try {
+ URIBuilder builder = new URIBuilder(uri);
+ builder.addParameter("$skip", String.valueOf(page*PER_PAGE));
+ List<JsonNode> pageData = new ArrayList<>();
+ for (JsonNode each: JerseyUtils.get(client, builder.build().toString(), logger))
+ pageData.add(each);
+ pageDataConsumer.consume(pageData);
+ if (pageData.size() < PER_PAGE)
+ break;
+ page++;
+ } catch (URISyntaxException|InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ String importIssues(Project project, String youTrackProject, ImportOption option,
+ boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {
+ logger.log("Importing issues from '" + youTrackProject + "'...");
+ Client client = newClient();
+ try {
+ String apiEndpoint = getApiEndpoint("/admin/projects?fields=id,name");
+ for (JsonNode projectNode: list(client, apiEndpoint, logger)) {
+ if (youTrackProject.equals(projectNode.get("name").asText())) {
+ ImportResult result = importIssues(projectNode.get("id").asText(),
+ project, retainIssueNumbers, option, dryRun, logger);
+ return result.toHtml("Issues imported successfully");
+ }
+ }
+ throw new ExplicitException("Unable to find YouTrack project: " + youTrackProject);
+ } finally {
+ client.close();
+ }
+ }
+
@Override
public boolean isValid(ConstraintValidatorContext context) {
Client client = ClientBuilder.newClient();
@@ -136,34 +1215,4 @@ public class ImportServer implements Serializable, Validatable {
return true;
}
- List<String> getProjectChoices() {
- List<String> choices = new ArrayList<>();
-
- Client client = newClient();
- try {
- TaskLogger logger = new TaskLogger() {
-
- @Override
- public void log(String message, String sessionId) {
- ImportServer.logger.info(message);
- }
-
- };
- String apiEndpoint = getApiEndpoint("/admin/projects?fields=name");
- for (JsonNode projectNode: ImportUtils.list(client, apiEndpoint, logger))
- choices.add(projectNode.get("name").asText());
- } finally {
- client.close();
- }
-
- Collections.sort(choices);
- return choices;
- }
-
- Client newClient() {
- Client client = ClientBuilder.newClient();
- client.register(HttpAuthenticationFeature.basic(getUserName(), getPassword()));
- return client;
- }
-
}
diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportUtils.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportUtils.java
deleted file mode 100644
index bdab8915c0..0000000000
--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ImportUtils.java
+++ /dev/null
@@ -1,935 +0,0 @@
-package io.onedev.server.plugin.imports.youtrack;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import javax.annotation.Nullable;
-import javax.ws.rs.client.Client;
-import javax.ws.rs.client.Invocation;
-import javax.ws.rs.client.WebTarget;
-import javax.ws.rs.core.Response;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.http.client.utils.URIBuilder;
-import org.unbescape.html.HtmlEscape;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.google.common.collect.Sets;
-
-import io.onedev.commons.utils.ExplicitException;
-import io.onedev.commons.utils.TaskLogger;
-import io.onedev.server.OneDev;
-import io.onedev.server.entitymanager.IssueLinkManager;
-import io.onedev.server.entitymanager.IssueManager;
-import io.onedev.server.entitymanager.LinkSpecManager;
-import io.onedev.server.entitymanager.SettingManager;
-import io.onedev.server.entitymanager.UserManager;
-import io.onedev.server.entityreference.ReferenceMigrator;
-import io.onedev.server.model.Issue;
-import io.onedev.server.model.IssueComment;
-import io.onedev.server.model.IssueField;
-import io.onedev.server.model.IssueLink;
-import io.onedev.server.model.IssueSchedule;
-import io.onedev.server.model.LinkSpec;
-import io.onedev.server.model.Project;
-import io.onedev.server.model.User;
-import io.onedev.server.model.support.LastUpdate;
-import io.onedev.server.model.support.administration.GlobalIssueSetting;
-import io.onedev.server.model.support.inputspec.InputSpec;
-import io.onedev.server.model.support.issue.StateSpec;
-import io.onedev.server.model.support.issue.field.spec.ChoiceField;
-import io.onedev.server.model.support.issue.field.spec.DateField;
-import io.onedev.server.model.support.issue.field.spec.DateTimeField;
-import io.onedev.server.model.support.issue.field.spec.FieldSpec;
-import io.onedev.server.model.support.issue.field.spec.FloatField;
-import io.onedev.server.model.support.issue.field.spec.IntegerField;
-import io.onedev.server.model.support.issue.field.spec.TextField;
-import io.onedev.server.model.support.issue.field.spec.UserChoiceField;
-import io.onedev.server.model.support.issue.field.spec.WorkingPeriodField;
-import io.onedev.server.persistence.dao.Dao;
-import io.onedev.server.security.SecurityUtils;
-import io.onedev.server.util.DateUtils;
-import io.onedev.server.util.JerseyUtils;
-import io.onedev.server.util.JerseyUtils.PageDataConsumer;
-import io.onedev.server.util.Pair;
-
-public class ImportUtils {
-
- static final String NAME = "YouTrack";
-
- static final int PER_PAGE = 50;
-
- static ImportOption buildImportOption(ImportServer server, Collection<JsonNode> projectNodes, TaskLogger logger) {
- ImportOption importOption = new ImportOption();
- Client client = server.newClient();
- try {
- Set<String> youTrackIssueStates = new LinkedHashSet<>();
- Set<String> youTrackIssueFields = new LinkedHashSet<>();
-
- for (JsonNode projectNode: projectNodes) {
- for (JsonNode customFieldNode: projectNode.get("customFields")) {
- String fieldName = customFieldNode.get("field").get("name").asText();
- JsonNode bundleNode = customFieldNode.get("bundle");
- if (bundleNode != null) {
- String bundleType = bundleNode.get("$type").asText();
- if (bundleType.equals("StateBundle")) {
- for (JsonNode valueNode: bundleNode.get("values"))
- youTrackIssueStates.add(valueNode.get("name").asText());
- } else if (bundleType.equals("EnumBundle") || bundleType.equals("OwnedBundle")) {
- for (JsonNode valueNode: bundleNode.get("values"))
- youTrackIssueFields.add(fieldName + "::" + valueNode.get("name").asText());
- } else {
- youTrackIssueFields.add(fieldName);
- }
- } else {
- youTrackIssueFields.add(fieldName);
- }
- }
- }
-
- for (String youTrackIssueState: youTrackIssueStates) {
- IssueStateMapping mapping = new IssueStateMapping();
- mapping.setYouTrackIssueState(youTrackIssueState);
- importOption.getIssueStateMappings().add(mapping);
- }
-
- for (String youTrackIssueField: youTrackIssueFields) {
- IssueFieldMapping mapping = new IssueFieldMapping();
- mapping.setYouTrackIssueField(youTrackIssueField);
- importOption.getIssueFieldMappings().add(mapping);
- }
-
- String apiEndpoint = server.getApiEndpoint("/issueTags?fields=name");
- for (JsonNode tagNode: list(client, apiEndpoint, logger)) {
- IssueTagMapping mapping = new IssueTagMapping();
- mapping.setYouTrackIssueTag(tagNode.get("name").asText());
- importOption.getIssueTagMappings().add(mapping);
- }
-
- apiEndpoint = server.getApiEndpoint("/issueLinkTypes?fields=name");
- for (JsonNode tagNode: list(client, apiEndpoint, logger)) {
- IssueLinkMapping mapping = new IssueLinkMapping();
- mapping.setYouTrackIssueLink(tagNode.get("name").asText());
- importOption.getIssueLinkMappings().add(mapping);
- }
-
- } finally {
- client.close();
- }
- return importOption;
- }
-
- static ImportResult importIssues(ImportServer server, String youTrackProjectId,
- Project oneDevProject, boolean retainIssueNumbers, ImportOption importOption, boolean dryRun, TaskLogger logger) {
- Client client = server.newClient();
- try {
- String apiEndpoint = server.getApiEndpoint("/admin/projects/" + youTrackProjectId + "?fields=shortName");
- String youTrackProjectShortName = JerseyUtils.get(client, apiEndpoint, logger).get("shortName").asText();
-
- Map<String, String> stateMappings = new HashMap<>();
- Map<String, Pair<FieldSpec, String>> fieldMappings = new HashMap<>();
- Map<String, Pair<FieldSpec, String>> tagMappings = new HashMap<>();
- Map<String, LinkSpec> linkMappings = new HashMap<>();
-
- Map<Long, Long> issueNumberMappings = new HashMap<>();
- Map<Long, Issue> issueMappings = new HashMap<>();
-
- for (IssueStateMapping mapping: importOption.getIssueStateMappings())
- stateMappings.put(mapping.getYouTrackIssueState(), mapping.getOneDevIssueState());
-
- for (IssueFieldMapping mapping: importOption.getIssueFieldMappings()) {
- String oneDevFieldName;
- String oneDevFieldValue;
- if (mapping.getOneDevIssueField().contains("::")) {
- oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");
- oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");
- } else {
- oneDevFieldName = mapping.getOneDevIssueField();
- oneDevFieldValue = null;
- }
- FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);
- if (fieldSpec == null)
- throw new ExplicitException("No field spec found: " + oneDevFieldName);
- fieldMappings.put(mapping.getYouTrackIssueField(), new Pair<>(fieldSpec, oneDevFieldValue));
- }
- for (IssueTagMapping mapping: importOption.getIssueTagMappings()) {
- String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");
- String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");
- FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);
- if (fieldSpec == null)
- throw new ExplicitException("No field spec found: " + oneDevFieldName);
- tagMappings.put(mapping.getYouTrackIssueTag(), new Pair<>(fieldSpec, oneDevFieldValue));
- }
- for (IssueLinkMapping mapping: importOption.getIssueLinkMappings()) {
- LinkSpec linkSpec = OneDev.getInstance(LinkSpecManager.class).find(mapping.getOneDevIssueLink());
- if (linkSpec == null)
- throw new ExplicitException("No link spec found: " + mapping.getOneDevIssueLink());
- linkMappings.put(mapping.getYouTrackIssueLink(), linkSpec);
- }
-
- Set<String> nonExistentLogins = new LinkedHashSet<>();
- Set<String> unmappedIssueTags = new LinkedHashSet<>();
- Set<String> unmappedIssueFields = new LinkedHashSet<>();
- Set<String> unmappedIssueLinks = new LinkedHashSet<>();
- Set<String> unmappedIssueStates = new LinkedHashSet<>();
- Map<String, String> mismatchedIssueFields = new LinkedHashMap<>();
- Set<String> tooLargeAttachments = new LinkedHashSet<>();
-
- AtomicInteger numOfImportedIssues = new AtomicInteger(0);
-
- List<Issue> issues = new ArrayList<>();
- Map<Long, Pair<LinkSpec, List<Long>>> issueLinkInfo = new HashMap<>();
- Map<Long, Set<Set<Long>>> processedSymmetricLinks = new HashMap<>();
-
- String fields = ""
- + "idReadable,"
- + "numberInProject,"
- + "summary,"
- + "description,"
- + "created,"
- + "comments(created,text,author(login,name,email),attachments(name,url,size)),"
- + "attachments(name,url,comment,size),"
- + "reporter(login,name,email),"
- + "tags(name),"
- + "customFields(name,value(name,login,email,presentation,text),projectCustomField(field(fieldType(id)))),"
- + "links(direction,linkType(name,sourceToTarget,targetToSource),issues(numberInProject))";
- PageDataConsumer pageDataConsumer = new PageDataConsumer() {
-
- @Nullable
- private String processAttachments(String issueUUID, String readableIssueId, @Nullable String markdown,
- List<JsonNode> attachmentNodes, Set<String> tooLargeAttachments) {
- if (markdown == null)
- markdown = "";
-
- Map<String, String> unreferencedAttachments = new LinkedHashMap<>();
-
- long maxUploadFileSize = OneDev.getInstance(SettingManager.class)
- .getPerformanceSetting().getMaxUploadFileSize()*1L*1024*1024;
- for (JsonNode attachmentNode: attachmentNodes) {
- String attachmentName = attachmentNode.get("name").asText(null);
- String attachmentUrl = attachmentNode.get("url").asText(null);
- long attachmentSize = attachmentNode.get("size").asLong(0);
- if (attachmentSize != 0 && attachmentName != null && attachmentUrl != null) {
- if (attachmentSize > maxUploadFileSize) {
- tooLargeAttachments.add(readableIssueId + ":" + attachmentName);
- } else {
- if (!attachmentUrl.startsWith("/api"))
- throw new ExplicitException("Unexpected attachment url: " + attachmentUrl);
- attachmentUrl = attachmentUrl.substring("/api".length());
-
- String endpoint = server.getApiEndpoint(attachmentUrl);
- WebTarget target = client.target(endpoint);
- Invocation.Builder builder = target.request();
- try (Response response = builder.get()) {
- String errorMessage = JerseyUtils.checkStatus(endpoint, response);
- if (errorMessage != null) {
- throw new ExplicitException(String.format(
- "Error downloading attachment (url: %s, error message: %s)",
- endpoint, errorMessage));
- }
- try (InputStream is = response.readEntity(InputStream.class)) {
- String oneDevAttachmentName = oneDevProject.saveAttachment(issueUUID, attachmentName, is);
- String oneDevAttachmentUrl = oneDevProject.getAttachmentUrlPath(issueUUID, oneDevAttachmentName);
- if (markdown.contains("(" + attachmentName + ")")) {
- markdown = markdown.replace("(" + attachmentName + ")", "(" + oneDevAttachmentUrl + ")");
- } else {
- unreferencedAttachments.put(attachmentName, oneDevAttachmentUrl);
- }
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
- }
- }
-
- if (!unreferencedAttachments.isEmpty()) {
- markdown += "\n\n";
- for (Map.Entry<String, String> entry: unreferencedAttachments.entrySet())
- markdown += "[" + entry.getKey() + "](" + entry.getValue() + ") ";
- }
-
- if (markdown.length() == 0)
- markdown = null;
-
- return markdown;
- }
-
- private String joinAsMultilineHtml(List<String> values) {
- List<String> escapedValues = new ArrayList<>();
- for (String value: values)
- escapedValues.add(HtmlEscape.escapeHtml5(value));
- return StringUtils.join(escapedValues, "<br>");
- }
-
- @Nullable
- private String getEmail(JsonNode userNode) {
- JsonNode emailNode = userNode.get("email");
- if (emailNode != null)
- return emailNode.asText(null);
- else
- return null;
- }
-
- @Override
- public void consume(List<JsonNode> pageData) throws InterruptedException {
- for (JsonNode issueNode: pageData) {
- if (Thread.interrupted())
- throw new InterruptedException();
-
- Issue issue = new Issue();
- String readableId = issueNode.get("idReadable").asText();
- Long oldNumber = issueNode.get("numberInProject").asLong();
- Long newNumber;
- if (dryRun || retainIssueNumbers)
- newNumber = oldNumber;
- else
- newNumber = getIssueManager().getNextNumber(oneDevProject);
- issue.setNumber(newNumber);
- issueNumberMappings.put(oldNumber, newNumber);
- issueMappings.put(oldNumber, issue);
- issue.setTitle(issueNode.get("summary").asText());
- issue.setDescription(issueNode.get("description").asText(null));
- issue.setSubmitDate(new Date(issueNode.get("created").asLong(System.currentTimeMillis())));
- issue.setProject(oneDevProject);
- issue.setNumberScope(oneDevProject.getForkRoot());
-
- if (!dryRun) {
- List<JsonNode> attachmentNodes = new ArrayList<>();
- for (JsonNode attachmentNode: issueNode.get("attachments")) {
- if (!attachmentNode.hasNonNull("comment"))
- attachmentNodes.add(attachmentNode);
- }
- issue.setDescription(processAttachments(issue.getUUID(), readableId,
- issue.getDescription(), attachmentNodes, tooLargeAttachments));
- }
-
- Map<String, String> extraIssueInfo = new LinkedHashMap<>();
-
- if (issueNode.hasNonNull("reporter")) {
- JsonNode reporterNode = issueNode.get("reporter");
- String email = getEmail(reporterNode);
- String login = reporterNode.get("login").asText();
- if (email != null) {
- User user = OneDev.getInstance(UserManager.class).findByVerifiedEmailAddress(email);
- if (user != null) {
- issue.setSubmitter(user);
- } else {
- issue.setSubmitter(OneDev.getInstance(UserManager.class).getUnknown());
- nonExistentLogins.add(login);
- }
- } else {
- issue.setSubmitter(OneDev.getInstance(UserManager.class).getUnknown());
- nonExistentLogins.add(login);
- }
- } else {
- issue.setSubmitter(SecurityUtils.getUser());
- }
-
- LastUpdate lastUpdate = new LastUpdate();
- lastUpdate.setActivity("Opened");
- lastUpdate.setDate(issue.getSubmitDate());
- lastUpdate.setUser(issue.getSubmitter());
- issue.setLastUpdate(lastUpdate);
-
- StateSpec initialState = getIssueSetting().getInitialStateSpec();
- for (JsonNode customFieldNode: issueNode.get("customFields")) {
- String fieldName = customFieldNode.get("name").asText();
- String fieldType = customFieldNode.get("$type").asText();
- switch (fieldType) {
- case "StateIssueCustomField":
- if (customFieldNode.hasNonNull("value")) {
- String state = customFieldNode.get("value").get("name").asText();
- String mappedState = stateMappings.get(state);
- if (mappedState != null)
- issue.setState(mappedState);
- else
- unmappedIssueStates.add(state);
- }
- break;
- case "SingleBuildIssueCustomField":
- case "SingleGroupIssueCustomField":
- case "SingleVersionIssueCustomField":
- String fieldValue = customFieldNode.get("value").asText(null);
- if (fieldValue != null) {
- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
- if (mapped == null) {
- unmappedIssueFields.add(fieldName);
- extraIssueInfo.put(fieldName, fieldValue);
- } else if (!(mapped.getFirst() instanceof TextField)) {
- mismatchedIssueFields.put(fieldName, "Should be mapped to a text field");
- extraIssueInfo.put(fieldName, fieldValue);
- } else {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(mapped.getFirst().getName());
- issueField.setType(InputSpec.TEXT);
- issueField.setValue(fieldValue);
- issue.getFields().add(issueField);
- }
- }
- break;
- case "MultiBuildIssueCustomField":
- case "MultiGroupIssueCustomField":
- case "MultiVersionIssueCustomField":
- List<String> fieldValues = new ArrayList<>();
- for (JsonNode valueNode: customFieldNode.get("value"))
- fieldValues.add(valueNode.get("name").asText());
- if (!fieldValues.isEmpty()) {
- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
- if (mapped == null) {
- unmappedIssueFields.add(fieldName);
- extraIssueInfo.put(fieldName, joinAsMultilineHtml(fieldValues));
- } else if (!(mapped.getFirst() instanceof TextField)) {
- mismatchedIssueFields.put(fieldName, "Should be mapped to a text field");
- extraIssueInfo.put(fieldName, joinAsMultilineHtml(fieldValues));
- } else {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(mapped.getFirst().getName());
- issueField.setType(InputSpec.TEXT);
- issueField.setValue(StringUtils.join(fieldValues, ", "));
- issue.getFields().add(issueField);
- }
- }
- break;
- case "SingleEnumIssueCustomField":
- case "SingleOwnedIssueCustomField":
- if (customFieldNode.hasNonNull("value")) {
- fieldValue = customFieldNode.get("value").get("name").asText();
- String fieldNameWithValue = fieldName + "::" + fieldValue;
- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldNameWithValue);
- if (mapped == null) {
- unmappedIssueFields.add(fieldNameWithValue);
- extraIssueInfo.put(fieldName, fieldValue);
- } else if (!(mapped.getFirst() instanceof ChoiceField) || mapped.getFirst().isAllowMultiple()) {
- mismatchedIssueFields.put(fieldNameWithValue, "Should be mapped to a single-valued enum field");
- extraIssueInfo.put(fieldName, fieldValue);
- } else {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(mapped.getFirst().getName());
- issueField.setType(InputSpec.ENUMERATION);
- issueField.setValue(mapped.getSecond());
- issue.getFields().add(issueField);
- }
- }
- break;
- case "MultiEnumIssueCustomField":
- case "MultiOwnedIssueCustomField":
- fieldValues = new ArrayList<>();
- for (JsonNode valueNode: customFieldNode.get("value"))
- fieldValues.add(valueNode.get("name").asText());
-
- boolean hasErrors = false;
- List<IssueField> issueFields = new ArrayList<>();
- for (String each: fieldValues) {
- String fieldNameWithValue = fieldName + "::" + each;
- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldNameWithValue);
- if (mapped == null) {
- unmappedIssueFields.add(fieldNameWithValue);
- hasErrors = true;
- } else if (!(mapped.getFirst() instanceof ChoiceField) || !mapped.getFirst().isAllowMultiple()) {
- mismatchedIssueFields.put(fieldNameWithValue, "Should be mapped to a multi-valued enum field");
- hasErrors = true;
- } else {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(mapped.getFirst().getName());
- issueField.setType(InputSpec.ENUMERATION);
- issueField.setValue(mapped.getSecond());
- issueFields.add(issueField);
- }
- }
- if (hasErrors)
- extraIssueInfo.put(fieldName, joinAsMultilineHtml(fieldValues));
- else
- issue.getFields().addAll(issueFields);
- break;
- case "SingleUserIssueCustomField":
- if (customFieldNode.hasNonNull("value")) {
- JsonNode valueNode = customFieldNode.get("value");
-
- String login = valueNode.get("login").asText();
- String fullName = valueNode.get("name").asText();
- String email = getEmail(valueNode);
-
- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
- if (mapped == null) {
- unmappedIssueFields.add(fieldName);
- extraIssueInfo.put(fieldName, fullName);
- } else if (!(mapped.getFirst() instanceof UserChoiceField) || mapped.getFirst().isAllowMultiple()) {
- mismatchedIssueFields.put(fieldName, "Should be mapped to a single-valued user field");
- extraIssueInfo.put(fieldName, fullName);
- } else {
- if (email != null) {
- User user = OneDev.getInstance(UserManager.class).findByVerifiedEmailAddress(email);
- if (user != null) {
- fieldValue = user.getName();
- } else {
- fieldValue = null;
- nonExistentLogins.add(login);
- }
- } else {
- fieldValue = null;
- nonExistentLogins.add(login);
- }
-
- if (fieldValue != null) {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(mapped.getFirst().getName());
- issueField.setType(InputSpec.USER);
- issueField.setValue(fieldValue);
- issue.getFields().add(issueField);
- }
- }
- }
- break;
- case "MultiUserIssueCustomField":
- List<String> fullNames = new ArrayList<>();
- for (JsonNode valueNode: customFieldNode.get("value"))
- fullNames.add(valueNode.get("name").asText());
- if (!fullNames.isEmpty()) {
- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
- if (mapped == null) {
- unmappedIssueFields.add(fieldName);
- extraIssueInfo.put(fieldName, joinAsMultilineHtml(fullNames));
- } else if (!(mapped.getFirst() instanceof UserChoiceField) || !mapped.getFirst().isAllowMultiple()) {
- mismatchedIssueFields.put(fieldName, "Should be mapped to a multi-valued user field");
- extraIssueInfo.put(fieldName, joinAsMultilineHtml(fullNames));
- } else {
- for (JsonNode valueNode: customFieldNode.get("value")) {
- String login = valueNode.get("login").asText();
- String email = getEmail(valueNode);
-
- if (email != null) {
- User user = OneDev.getInstance(UserManager.class).findByVerifiedEmailAddress(email);
- if (user != null) {
- fieldValue = user.getName();
- } else {
- fieldValue = null;
- nonExistentLogins.add(login);
- }
- } else {
- fieldValue = null;
- nonExistentLogins.add(login);
- }
-
- if (fieldValue != null) {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(mapped.getFirst().getName());
- issueField.setType(InputSpec.USER);
- issueField.setValue(fieldValue);
- issue.getFields().add(issueField);
- }
- }
- }
- }
- break;
- case "DateIssueCustomField":
- if (customFieldNode.hasNonNull("value")) {
- Date date = new Date(customFieldNode.get("value").asLong());
- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
- if (mapped == null) {
- unmappedIssueFields.add(fieldName);
- extraIssueInfo.put(fieldName, DateUtils.formatDate(date));
- } else if (!(mapped.getFirst() instanceof DateField)) {
- mismatchedIssueFields.put(fieldName, "Should be mapped to a date field");
- extraIssueInfo.put(fieldName, DateUtils.formatDate(date));
- } else {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(mapped.getFirst().getName());
- issueField.setType(InputSpec.DATE);
- issueField.setValue(DateUtils.formatDate(date));
- issue.getFields().add(issueField);
- }
- }
- break;
- case "SimpleIssueCustomField":
- if (customFieldNode.hasNonNull("value")) {
- JsonNode projectCustomField = customFieldNode.get("projectCustomField");
- JsonNode fieldNode = projectCustomField.get("field");
- if (fieldNode != null) {
- switch (fieldNode.get("fieldType").get("id").asText()) {
- case "date":
- Date date = new Date(customFieldNode.get("value").asLong());
- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
- if (mapped == null) {
- unmappedIssueFields.add(fieldName);
- extraIssueInfo.put(fieldName, DateUtils.formatDateTime(date));
- } else if (!(mapped.getFirst() instanceof DateTimeField)) {
- mismatchedIssueFields.put(fieldName, "Should be mapped to a datetime field");
- extraIssueInfo.put(fieldName, DateUtils.formatDateTime(date));
- } else {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(mapped.getFirst().getName());
- issueField.setType(InputSpec.DATE_TIME);
- issueField.setValue(DateUtils.formatDateTime(date));
- issue.getFields().add(issueField);
- }
- break;
- case "float":
- fieldValue = customFieldNode.get("value").asText();
- mapped = fieldMappings.get(fieldName);
- if (mapped == null) {
- unmappedIssueFields.add(fieldName);
- extraIssueInfo.put(fieldName, fieldValue);
- } else if (!(mapped.getFirst() instanceof FloatField)) {
- mismatchedIssueFields.put(fieldName, "Should be mapped to a float field");
- extraIssueInfo.put(fieldName, fieldValue);
- } else {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(mapped.getFirst().getName());
- issueField.setType(InputSpec.FLOAT);
- issueField.setValue(fieldValue);
- issue.getFields().add(issueField);
- }
- break;
- case "integer":
- fieldValue = customFieldNode.get("value").asText();
- mapped = fieldMappings.get(fieldName);
- if (mapped == null) {
- unmappedIssueFields.add(fieldName);
- extraIssueInfo.put(fieldName, fieldValue);
- } else if (!(mapped.getFirst() instanceof IntegerField)) {
- mismatchedIssueFields.put(fieldName, "Should be mapped to an integer field");
- extraIssueInfo.put(fieldName, fieldValue);
- } else {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(mapped.getFirst().getName());
- issueField.setType(InputSpec.INTEGER);
- issueField.setValue(fieldValue);
- issue.getFields().add(issueField);
- }
- break;
- case "string":
- fieldValue = customFieldNode.get("value").asText();
- mapped = fieldMappings.get(fieldName);
- if (mapped == null) {
- unmappedIssueFields.add(fieldName);
- extraIssueInfo.put(fieldName, fieldValue);
- } else if (!(mapped.getFirst() instanceof TextField)) {
- mismatchedIssueFields.put(fieldName, "Should be mapped to a text field");
- extraIssueInfo.put(fieldName, fieldValue);
- } else {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(mapped.getFirst().getName());
- issueField.setType(InputSpec.TEXT);
- issueField.setValue(fieldValue);
- issue.getFields().add(issueField);
- }
- break;
- }
- }
- }
- break;
- case "TextIssueCustomField":
- if (customFieldNode.hasNonNull("value")) {
- fieldValue = customFieldNode.get("value").get("text").asText();
- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
- if (mapped == null) {
- unmappedIssueFields.add(fieldName);
- extraIssueInfo.put(fieldName, fieldValue);
- } else if (!(mapped.getFirst() instanceof TextField)
- || !((TextField)mapped.getFirst()).isMultiline()) {
- mismatchedIssueFields.put(fieldName, "Should be mapped to a multi-line text field");
- extraIssueInfo.put(fieldName, fieldValue);
- } else {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(mapped.getFirst().getName());
- issueField.setType(InputSpec.TEXT);
- issueField.setValue(fieldValue);
- issue.getFields().add(issueField);
- }
- }
- break;
- case "PeriodIssueCustomField":
- if (customFieldNode.hasNonNull("value")) {
- fieldValue = customFieldNode.get("value").get("presentation").asText();
- Pair<FieldSpec, String> mapped = fieldMappings.get(fieldName);
- if (mapped == null) {
- unmappedIssueFields.add(fieldName);
- extraIssueInfo.put(fieldName, fieldValue);
- } else if (!(mapped.getFirst() instanceof WorkingPeriodField)) {
- mismatchedIssueFields.put(fieldName, "Should be mapped to a working period field");
- extraIssueInfo.put(fieldName, fieldValue);
- } else {
- IssueField issueField = new IssueField();
- issueField.setIssue(issue);
- issueField.setName(mapped.getFirst().getName());
- issueField.setType(InputSpec.WORKING_PERIOD);
- issueField.setValue(fieldValue);
- issue.getFields().add(issueField);
- }
- }
- break;
- }
- }
- if (issue.getState() == null)
- issue.setState(initialState.getName());
-
- for (JsonNode linkNode: issueNode.get("links")) {
- List<Long> linkedIssueNumbers = new ArrayList<>();
- for (JsonNode linkedIssueNode: linkNode.get("issues"))
- linkedIssueNumbers.add(linkedIssueNode.get("numberInProject").asLong());
-
- if (!linkedIssueNumbers.isEmpty() && linkNode.hasNonNull("linkType")) {
- JsonNode linkTypeNode = linkNode.get("linkType");
- String direction = linkNode.get("direction").asText();
- String linkName = linkTypeNode.get("name").asText(null);
- if (linkName != null) {
- LinkSpec linkSpec = linkMappings.get(linkName);
- if (linkSpec == null) {
- List<String> linkedIssueLinks = new ArrayList<>();
- for (Long issueNumber: linkedIssueNumbers)
- linkedIssueLinks.add(youTrackProjectShortName + "-" + issueNumber);
-
- unmappedIssueLinks.add(linkName);
- switch (direction) {
- case "BOTH":
- case "OUTWARD":
- linkName = linkTypeNode.get("sourceToTarget").asText(null);
- if (linkName != null)
- extraIssueInfo.put(linkName, joinAsMultilineHtml(linkedIssueLinks));
- break;
- case "INWARD":
- linkName = linkTypeNode.get("targetToSource").asText(null);
- if (linkName != null)
- extraIssueInfo.put(linkName, joinAsMultilineHtml(linkedIssueLinks));
- break;
- }
- } else if ("OUTWARD".equals(direction)) {
- issueLinkInfo.put(oldNumber, new Pair<>(linkSpec, linkedIssueNumbers));
- } else if ("BOTH".equals(direction)) {
- Set<Set<Long>> value = processedSymmetricLinks.get(linkSpec.getId());
- if (value == null) {
- value = new HashSet<>();
- processedSymmetricLinks.put(linkSpec.getId(), value);
- }
- List<Long> filteredIssueNumbers = new ArrayList<>();
- for (Long issueNumber: linkedIssueNumbers) {
- Set<Long> linkSides = Sets.newHashSet(oldNumber, issueNumber);
- if (value.add(linkSides))
- filteredIssueNumbers.add(issueNumber);
- }
- if (!filteredIssueNumbers.isEmpty())
- issueLinkInfo.put(oldNumber, new Pair<>(linkSpec, filteredIssueNumbers));
- }
- }
- }
- }
-
- List<String> currentUnmappedTags = new ArrayList<>();
- for (JsonNode tagNode: issueNode.get("tags")) {
- String tagName = tagNode.get("name").asText();
- Pair<FieldSpec, String> mapped = tagMappings.get(tagName);
- if (mapped != null) {
- IssueField tagField = new IssueField();
- tagField.setIssue(issue);
- tagField.setName(mapped.getFirst().getName());
- tagField.setType(InputSpec.ENUMERATION);
- tagField.setValue(mapped.getSecond());
- tagField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));
- issue.getFields().add(tagField);
- } else {
- currentUnmappedTags.add(HtmlEscape.escapeHtml5(tagName));
- unmappedIssueTags.add(HtmlEscape.escapeHtml5(tagName));
- }
- }
- if (!currentUnmappedTags.isEmpty())
- extraIssueInfo.put("Tags", joinAsMultilineHtml(currentUnmappedTags));
-
- for (JsonNode commentNode: issueNode.get("comments")) {
- String commentContent = commentNode.get("text").asText(null);
- if (commentContent != null || !commentNode.get("attachments").isEmpty()) {
- IssueComment comment = new IssueComment();
- comment.setIssue(issue);
- if (!dryRun) {
- List<JsonNode> attachmentNodes = new ArrayList<>();
- for (JsonNode attachmentNode: commentNode.get("attachments"))
- attachmentNodes.add(attachmentNode);
- comment.setContent(processAttachments(issue.getUUID(), readableId,
- commentContent, attachmentNodes, tooLargeAttachments));
- }
- comment.setDate(new Date(commentNode.get("created").asLong(System.currentTimeMillis())));
- if (commentNode.hasNonNull("author")) {
- JsonNode authorNode = commentNode.get("author");
- String email = getEmail(authorNode);
- String login = authorNode.get("login").asText();
- if (email != null) {
- User user = OneDev.getInstance(UserManager.class).findByVerifiedEmailAddress(email);
- if (user != null) {
- comment.setUser(user);
- } else {
- comment.setUser(OneDev.getInstance(UserManager.class).getUnknown());
- nonExistentLogins.add(login);
- }
- } else {
- comment.setUser(OneDev.getInstance(UserManager.class).getUnknown());
- nonExistentLogins.add(login);
- }
- } else {
- issue.setSubmitter(SecurityUtils.getUser());
- }
- issue.getComments().add(comment);
- }
- }
-
- issue.setCommentCount(issue.getComments().size());
-
- Set<String> fieldAndValues = new HashSet<>();
- for (IssueField field: issue.getFields()) {
- String fieldAndValue = field.getName() + "::" + field.getValue();
- if (!fieldAndValues.add(fieldAndValue)) {
- String errorMessage = String.format(
- "Duplicate issue field mapping (issue: %s, field: %s)",
- readableId, fieldAndValue);
- throw new ExplicitException(errorMessage);
- }
- }
-
- if (!extraIssueInfo.isEmpty()) {
- StringBuilder builder = new StringBuilder("|");
- for (String key: extraIssueInfo.keySet())
- builder.append(key).append("|");
- builder.append("\n|");
- extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));
- builder.append("\n|");
- for (String value: extraIssueInfo.values())
- builder.append(value).append("|");
-
- if (issue.getDescription() != null)
- issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());
- else
- issue.setDescription(builder.toString());
- }
- issues.add(issue);
- }
- logger.log("Imported " + numOfImportedIssues.addAndGet(pageData.size()) + " issues");
- }
-
- };
-
- apiEndpoint = server.getApiEndpoint("/admin/projects/" + youTrackProjectId + "/issues?fields=" + fields);
- list(client, apiEndpoint, pageDataConsumer, logger);
-
- if (!dryRun) {
- ReferenceMigrator migrator = new ReferenceMigrator(Issue.class, issueNumberMappings);
- Dao dao = OneDev.getInstance(Dao.class);
- for (Issue issue: issues) {
- if (issue.getDescription() != null)
- issue.setDescription(migrator.migratePrefixed(issue.getDescription(), youTrackProjectShortName + "-"));
-
- getIssueManager().save(issue);
- for (IssueSchedule schedule: issue.getSchedules())
- dao.persist(schedule);
- for (IssueField field: issue.getFields())
- dao.persist(field);
- for (IssueComment comment: issue.getComments()) {
- comment.setContent(migrator.migratePrefixed(comment.getContent(), youTrackProjectShortName + "-"));
- dao.persist(comment);
- }
- }
-
- for (Map.Entry<Long, Pair<LinkSpec, List<Long>>> entry: issueLinkInfo.entrySet()) {
- Issue source = issueMappings.get(entry.getKey());
- if (source != null) {
- for (Long targetNumber: entry.getValue().getSecond()) {
- Issue target = issueMappings.get(targetNumber);
- if (target != null) {
- IssueLink link = new IssueLink();
- link.setSource(source);
- link.setTarget(target);
- link.setSpec(entry.getValue().getFirst());
- OneDev.getInstance(IssueLinkManager.class).save(link);
- }
- }
- }
- }
- }
-
- ImportResult result = new ImportResult();
- result.mismatchedIssueFields.putAll(mismatchedIssueFields);
- result.nonExistentLogins.addAll(nonExistentLogins);
- result.tooLargeAttachments.addAll(tooLargeAttachments);
- result.unmappedIssueFields.addAll(unmappedIssueFields);
- result.unmappedIssueStates.addAll(unmappedIssueStates);
- result.unmappedIssueLinks.addAll(unmappedIssueLinks);
- result.unmappedIssueTags.addAll(unmappedIssueTags);
-
- return result;
- } finally {
- client.close();
- }
- }
-
- private static IssueManager getIssueManager() {
- return OneDev.getInstance(IssueManager.class);
- }
-
- static GlobalIssueSetting getIssueSetting() {
- return OneDev.getInstance(SettingManager.class).getIssueSetting();
- }
-
- static List<JsonNode> list(Client client, String apiEndpoint, TaskLogger logger) {
- List<JsonNode> result = new ArrayList<>();
- list(client, apiEndpoint, new PageDataConsumer() {
-
- @Override
- public void consume(List<JsonNode> pageData) {
- result.addAll(pageData);
- }
-
- }, logger);
- return result;
- }
-
- static void list(Client client, String apiEndpoint, PageDataConsumer pageDataConsumer, TaskLogger logger) {
- URI uri;
- try {
- uri = new URIBuilder(apiEndpoint)
- .addParameter("$top", String.valueOf(PER_PAGE)).build();
- } catch (URISyntaxException e) {
- throw new RuntimeException(e);
- }
-
- int page = 0;
- while (true) {
- try {
- URIBuilder builder = new URIBuilder(uri);
- builder.addParameter("$skip", String.valueOf(page*PER_PAGE));
- List<JsonNode> pageData = new ArrayList<>();
- for (JsonNode each: JerseyUtils.get(client, builder.build().toString(), logger))
- pageData.add(each);
- pageDataConsumer.consume(pageData);
- if (pageData.size() < PER_PAGE)
- break;
- page++;
- } catch (URISyntaxException|InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
-}
\ No newline at end of file
diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ProjectMapping.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ProjectMapping.java
index b693e17195..c77c00b07e 100644
--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ProjectMapping.java
+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/ProjectMapping.java
@@ -5,10 +5,11 @@ import java.util.List;
import org.hibernate.validator.constraints.NotEmpty;
+import io.onedev.server.util.ComponentContext;
import io.onedev.server.util.validation.annotation.ProjectPath;
+import io.onedev.server.web.editable.BeanEditor;
import io.onedev.server.web.editable.annotation.ChoiceProvider;
import io.onedev.server.web.editable.annotation.Editable;
-import io.onedev.server.web.util.WicketUtils;
@Editable
public class ProjectMapping implements Serializable {
@@ -47,7 +48,9 @@ public class ProjectMapping implements Serializable {
@SuppressWarnings("unused")
private static List<String> getProjectChoices() {
- return WicketUtils.getPage().getMetaData(ImportServer.META_DATA_KEY).getProjectChoices();
+ BeanEditor editor = ComponentContext.get().getComponent().findParent(BeanEditor.class);
+ ImportProjects setting = (ImportProjects) editor.getModelObject();
+ return setting.server.listProjects();
}
}
\ No newline at end of file
diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImporter.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImporter.java
index dad379ccf7..8818266be9 100644
--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImporter.java
+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackIssueImporter.java
@@ -1,73 +1,88 @@
package io.onedev.server.plugin.imports.youtrack;
-import static io.onedev.server.plugin.imports.youtrack.ImportUtils.NAME;
-import static io.onedev.server.plugin.imports.youtrack.ImportUtils.importIssues;
-import static io.onedev.server.plugin.imports.youtrack.ImportUtils.list;
-
-import java.util.ArrayList;
+import java.io.Serializable;
import java.util.List;
-import javax.ws.rs.client.Client;
-
-import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.collect.Lists;
-import io.onedev.commons.utils.ExplicitException;
import io.onedev.commons.utils.TaskLogger;
import io.onedev.server.imports.IssueImporter;
import io.onedev.server.model.Project;
-import io.onedev.server.web.util.WicketUtils;
+import io.onedev.server.web.util.ImportStep;
-public class YouTrackIssueImporter extends IssueImporter<ImportServer, IssueImportSource, ImportOption> {
+public class YouTrackIssueImporter implements IssueImporter {
private static final long serialVersionUID = 1L;
+ private final ImportStep<ImportServer> serverStep = new ImportStep<ImportServer>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Authenticate to YouTrack";
+ }
+
+ @Override
+ protected ImportServer newSetting() {
+ return new ImportServer();
+ }
+
+ };
+
+ private final ImportStep<ImportProject> projectStep = new ImportStep<ImportProject>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Choose project";
+ }
+
+ @Override
+ protected ImportProject newSetting() {
+ ImportProject project = new ImportProject();
+ project.server = serverStep.getSetting();
+ return project;
+ }
+
+ };
+
+ private final ImportStep<ImportOption> optionStep = new ImportStep<ImportOption>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Specify import option";
+ }
+
+ @Override
+ protected ImportOption newSetting() {
+ ImportServer server = serverStep.getSetting();
+ String project = projectStep.getSetting().getProject();
+ return server.buildImportOption(project);
+ }
+
+ };
+
@Override
public String getName() {
- return NAME;
+ return YouTrackPluginModule.NAME;
}
@Override
- public String doImport(ImportServer where, IssueImportSource what, ImportOption how, Project project,
- boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {
- logger.log("Importing issues from '" + what.getProject() + "'...");
- Client client = where.newClient();
- try {
- String apiEndpoint = where.getApiEndpoint("/admin/projects?fields=id,name");
- for (JsonNode projectNode: list(client, apiEndpoint, logger)) {
- if (what.getProject().equals(projectNode.get("name").asText())) {
- ImportResult result = importIssues(where, projectNode.get("id").asText(),
- project, retainIssueNumbers, how, dryRun, logger);
- return result.toHtml("Issues imported successfully");
- }
- }
- throw new ExplicitException("Unable to find YouTrack project: " + what.getProject());
- } finally {
- client.close();
- }
+ public String doImport(Project project, boolean retainIssueNumbers, boolean dryRun, TaskLogger logger) {
+ ImportServer server = serverStep.getSetting();
+ String youTrackProject = projectStep.getSetting().getProject();
+ ImportOption option = optionStep.getSetting();
+
+ return server.importIssues(project, youTrackProject, option, retainIssueNumbers, dryRun, logger);
}
@Override
- public IssueImportSource getWhat(ImportServer where, TaskLogger logger) {
- WicketUtils.getPage().setMetaData(ImportServer.META_DATA_KEY, where);
- return new IssueImportSource();
+ public List<ImportStep<? extends Serializable>> getSteps() {
+ return Lists.newArrayList(serverStep, projectStep, optionStep);
}
- @Override
- public ImportOption getHow(ImportServer where, IssueImportSource what, TaskLogger logger) {
- Client client = where.newClient();
- try {
- String apiEndpoint = where.getApiEndpoint("/admin/projects?fields=id,name,customFields(field(name),bundle(values(name)))");
- for (JsonNode projectNode: list(client, apiEndpoint, logger)) {
- if (what.getProject().equals(projectNode.get("name").asText())) {
- List<JsonNode> projectNodes = new ArrayList<>();
- projectNodes.add(projectNode);
- return ImportUtils.buildImportOption(where, projectNodes, logger);
- }
- }
- throw new ExplicitException("Unable to find YouTrack project: " + what.getProject());
- } finally {
- client.close();
- }
- }
-
}
\ No newline at end of file
diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackPluginModule.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackPluginModule.java
index c7482489dc..b9e4de1089 100644
--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackPluginModule.java
+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackPluginModule.java
@@ -1,6 +1,5 @@
package io.onedev.server.plugin.imports.youtrack;
-import java.io.Serializable;
import java.util.Collection;
import com.google.common.collect.Lists;
@@ -17,6 +16,8 @@ import io.onedev.server.imports.ProjectImporterContribution;
*/
public class YouTrackPluginModule extends AbstractPluginModule {
+ static final String NAME = "YouTrack";
+
@Override
protected void configure() {
super.configure();
@@ -25,7 +26,7 @@ public class YouTrackPluginModule extends AbstractPluginModule {
contribute(ProjectImporterContribution.class, new ProjectImporterContribution() {
@Override
- public Collection<ProjectImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> getImporters() {
+ public Collection<ProjectImporter> getImporters() {
return Lists.newArrayList(new YouTrackProjectImporter());
}
@@ -39,7 +40,7 @@ public class YouTrackPluginModule extends AbstractPluginModule {
contribute(IssueImporterContribution.class, new IssueImporterContribution() {
@Override
- public Collection<IssueImporter<? extends Serializable, ? extends Serializable, ? extends Serializable>> getImporters() {
+ public Collection<IssueImporter> getImporters() {
return Lists.newArrayList(new YouTrackIssueImporter());
}
diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImporter.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImporter.java
index 533aa537a9..2aefd4c1a7 100644
--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImporter.java
+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackProjectImporter.java
@@ -1,119 +1,86 @@
package io.onedev.server.plugin.imports.youtrack;
-import static io.onedev.server.plugin.imports.youtrack.ImportUtils.list;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
+import java.io.Serializable;
import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import javax.ws.rs.client.Client;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
-import io.onedev.commons.utils.ExceptionUtils;
-import io.onedev.commons.utils.ExplicitException;
import io.onedev.commons.utils.TaskLogger;
-import io.onedev.server.OneDev;
-import io.onedev.server.entitymanager.ProjectManager;
import io.onedev.server.imports.ProjectImporter;
-import io.onedev.server.model.Project;
-import io.onedev.server.storage.StorageManager;
-import io.onedev.server.web.util.WicketUtils;
+import io.onedev.server.web.util.ImportStep;
-public class YouTrackProjectImporter extends ProjectImporter<ImportServer, ProjectImportSource, ImportOption> {
+public class YouTrackProjectImporter implements ProjectImporter {
private static final long serialVersionUID = 1L;
+ private final ImportStep<ImportServer> serverStep = new ImportStep<ImportServer>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Authenticate to YouTrack";
+ }
+
+ @Override
+ protected ImportServer newSetting() {
+ return new ImportServer();
+ }
+
+ };
+
+ private final ImportStep<ImportProjects> projectsStep = new ImportStep<ImportProjects>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Specify projects";
+ }
+
+ @Override
+ protected ImportProjects newSetting() {
+ ImportProjects projects = new ImportProjects();
+ projects.server = serverStep.getSetting();
+ return projects;
+ }
+
+ };
+
+ private final ImportStep<ImportOption> optionStep = new ImportStep<ImportOption>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getTitle() {
+ return "Specify import option";
+ }
+
+ @Override
+ protected ImportOption newSetting() {
+ ImportServer server = serverStep.getSetting();
+ ImportProjects projects = projectsStep.getSetting();
+ return server.buildImportOption(projects);
+ }
+
+ };
+
@Override
public String getName() {
- return ImportUtils.NAME;
+ return YouTrackPluginModule.NAME;
}
@Override
- public String doImport(ImportServer where, ProjectImportSource what, ImportOption how,
- boolean dryRun, TaskLogger logger) {
- Map<String, String> youTrackProjectIds = new HashMap<>();
- Map<String, String> youTrackProjectDescriptions = new HashMap<>();
- Collection<Long> projectIds = new ArrayList<>();
-
- Client client = where.newClient();
- try {
- String apiEndpoint = where.getApiEndpoint("/admin/projects?fields=id,name,description");
- for (JsonNode projectNode: list(client, apiEndpoint, logger)) {
- String projectName = projectNode.get("name").asText();
- youTrackProjectIds.put(projectName, projectNode.get("id").asText());
- youTrackProjectDescriptions.put(projectName, projectNode.get("description").asText(null));
- }
-
- ImportResult result = new ImportResult();
-
- for (ProjectMapping projectMapping: what.getProjectMappings()) {
- logger.log("Importing project '" + projectMapping.getYouTrackProject() + "'...");
-
- String youTrackProjectId = youTrackProjectIds.get(projectMapping.getYouTrackProject());
- if (youTrackProjectId == null)
- throw new ExplicitException("Unable to find YouTrack project: " + projectMapping.getYouTrackProject());
-
- ProjectManager projectManager = OneDev.getInstance(ProjectManager.class);
- Project project = projectManager.initialize(projectMapping.getOneDevProject());
- Preconditions.checkState(project.isNew());
- project.setDescription(youTrackProjectDescriptions.get(projectMapping.getYouTrackProject()));
- project.setIssueManagement(true);
-
- if (!dryRun) {
- projectManager.create(project);
- projectIds.add(project.getId());
- }
-
- ImportResult currentResult = ImportUtils.importIssues(where, youTrackProjectId, project,
- true, how, dryRun, logger);
- result.mismatchedIssueFields.putAll(currentResult.mismatchedIssueFields);
- result.nonExistentLogins.addAll(currentResult.nonExistentLogins);
- result.tooLargeAttachments.addAll(currentResult.tooLargeAttachments);
- result.unmappedIssueFields.addAll(currentResult.unmappedIssueFields);
- result.unmappedIssueStates.addAll(currentResult.unmappedIssueStates);
- result.unmappedIssueLinks.addAll(currentResult.unmappedIssueLinks);
- result.unmappedIssueTags.addAll(currentResult.unmappedIssueTags);
- }
-
- return result.toHtml("Projects imported successfully");
- } catch (Exception e) {
- for (Long projectId: projectIds)
- OneDev.getInstance(StorageManager.class).deleteProjectDir(projectId);
- throw ExceptionUtils.unchecked(e);
- } finally {
- client.close();
- }
+ public String doImport(boolean dryRun, TaskLogger logger) {
+ ImportServer server = serverStep.getSetting();
+ ImportProjects projects = projectsStep.getSetting();
+ ImportOption option = optionStep.getSetting();
+ return server.importProjects(projects, option, dryRun, logger);
}
@Override
- public ProjectImportSource getWhat(ImportServer where, TaskLogger logger) {
- WicketUtils.getPage().setMetaData(ImportServer.META_DATA_KEY, where);
- return new ProjectImportSource();
+ public List<ImportStep<? extends Serializable>> getSteps() {
+ return Lists.newArrayList(serverStep, projectsStep, optionStep);
}
- @Override
- public ImportOption getHow(ImportServer where, ProjectImportSource what, TaskLogger logger) {
- Client client = where.newClient();
- try {
- List<JsonNode> projectNodes = new ArrayList<>();
- String apiEndpoint = where.getApiEndpoint("/admin/projects?fields=id,name,customFields(field(name),bundle(values(name)))");
- Set<String> projectNames = what.getProjectMappings().stream()
- .map(it->it.getYouTrackProject()).collect(Collectors.toSet());
- for (JsonNode projectNode: list(client, apiEndpoint, logger)) {
- if (projectNames.contains(projectNode.get("name").asText()))
- projectNodes.add(projectNode);
- }
-
- return ImportUtils.buildImportOption(where, projectNodes, logger);
- } finally {
- client.close();
- }
- }
-
}
\ No newline at end of file
diff --git a/server-product/pom.xml b/server-product/pom.xml
index 6fb6f0d8c1..a5682ee894 100644
--- a/server-product/pom.xml
+++ b/server-product/pom.xml
@@ -127,17 +127,17 @@
</dependency>
<dependency>
<groupId>io.onedev</groupId>
- <artifactId>server-plugin-import-youtrack</artifactId>
+ <artifactId>server-plugin-import-bitbucketcloud</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.onedev</groupId>
- <artifactId>server-plugin-import-jiracloud</artifactId>
+ <artifactId>server-plugin-import-youtrack</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.onedev</groupId>
- <artifactId>server-plugin-import-bitbucketcloud</artifactId>
+ <artifactId>server-plugin-import-jiracloud</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>