The Unified and Autonomous Development Platform
Improve GitHub importer
Robin Shenfc7669f73f9d
diff --git a/server-core/src/main/java/io/onedev/server/model/Project.java b/server-core/src/main/java/io/onedev/server/model/Project.javaindex 96992f3775..5e792d5f11 100644--- a/server-core/src/main/java/io/onedev/server/model/Project.java+++ b/server-core/src/main/java/io/onedev/server/model/Project.java@@ -113,6 +113,7 @@ import io.onedev.server.persistence.TransactionManager;import io.onedev.server.security.SecurityUtils;import io.onedev.server.storage.AttachmentStorageManager;import io.onedev.server.storage.StorageManager;+import io.onedev.server.util.AttachmentTooLargeException;import io.onedev.server.util.CollectionUtils;import io.onedev.server.util.ComponentContext;import io.onedev.server.util.NameAware;@@ -1521,7 +1522,7 @@ public class Project extends AbstractEntity implements NameAware {while (-1 != (n = attachmentStream.read(buffer))) {count += n;if (count > MAX_ATTACHMENT_SIZE) {- throw new RuntimeException("Upload must be less than "+ throw new AttachmentTooLargeException("Upload must be less than "+ FileUtils.byteCountToDisplaySize(MAX_ATTACHMENT_SIZE));}os.write(buffer, 0, n);diff --git a/server-core/src/main/java/io/onedev/server/util/AttachmentTooLargeException.java b/server-core/src/main/java/io/onedev/server/util/AttachmentTooLargeException.javanew file mode 100644index 0000000000..a6a8e47db6--- /dev/null+++ b/server-core/src/main/java/io/onedev/server/util/AttachmentTooLargeException.java@@ -0,0 +1,13 @@+package io.onedev.server.util;++import io.onedev.commons.utils.ExplicitException;++public class AttachmentTooLargeException extends ExplicitException {++ private static final long serialVersionUID = 1L;++ public AttachmentTooLargeException(String message) {+ super(message);+ }++}diff --git a/server-core/src/main/java/io/onedev/server/util/JerseyUtils.java b/server-core/src/main/java/io/onedev/server/util/JerseyUtils.javanew file mode 100644index 0000000000..2fdcd7d008--- /dev/null+++ b/server-core/src/main/java/io/onedev/server/util/JerseyUtils.java@@ -0,0 +1,26 @@+package io.onedev.server.util;++import javax.annotation.Nullable;+import javax.ws.rs.core.Response;++import io.onedev.commons.utils.StringUtils;++public class JerseyUtils {++ @Nullable+ public static String checkStatus(Response response) {+ int status = response.getStatus();+ if (status != 200) {+ String errorMessage = response.readEntity(String.class);+ if (StringUtils.isNotBlank(errorMessage)) {+ return String.format("Http request failed (status code: %d, error message: %s)",+ status, errorMessage);+ } else {+ return String.format("Http request failed (status code: %d)", status);+ }+ } else {+ return null;+ }+ }++}diff --git a/server-core/src/main/java/io/onedev/server/util/markdown/DefaultMarkdownManager.java b/server-core/src/main/java/io/onedev/server/util/markdown/DefaultMarkdownManager.javaindex ee3bdb2ba3..379208446b 100644--- a/server-core/src/main/java/io/onedev/server/util/markdown/DefaultMarkdownManager.java+++ b/server-core/src/main/java/io/onedev/server/util/markdown/DefaultMarkdownManager.java@@ -42,8 +42,7 @@ public class DefaultMarkdownManager implements MarkdownManager {this.htmlTransformers = htmlTransformers;}- @Override- public String render(String markdown) {+ private MutableDataHolder setupOptions() {List<Extension> extensions = new ArrayList<>();extensions.add(AnchorLinkExtension.create());extensions.add(TablesExtension.create());@@ -53,7 +52,7 @@ public class DefaultMarkdownManager implements MarkdownManager {extensions.add(AutolinkExtension.create());extensions.addAll(contributedExtensions);- MutableDataHolder options = new MutableDataSet()+ return new MutableDataSet().set(HtmlRenderer.GENERATE_HEADER_ID, true).set(AnchorLinkExtension.ANCHORLINKS_SET_NAME, true).set(AnchorLinkExtension.ANCHORLINKS_WRAP_TEXT, false)@@ -65,13 +64,13 @@ public class DefaultMarkdownManager implements MarkdownManager {.set(TablesExtension.DISCARD_EXTRA_COLUMNS, true).set(TablesExtension.HEADER_SEPARATOR_COLUMN_MATCH, true).set(Parser.EXTENSIONS, extensions);-- Parser parser = Parser.builder(options).build();-- HtmlRenderer htmlRenderer = HtmlRenderer.builder(options).build();-- Node document = parser.parse(markdown);- return htmlRenderer.render(document);+ }++ @Override+ public String render(String markdown) {+ MutableDataHolder options = setupOptions();+ Node node = parse(markdown);+ return HtmlRenderer.builder(options).build().render(node);}@Override@@ -94,4 +93,11 @@ public class DefaultMarkdownManager implements MarkdownManager {return process(HtmlUtils.parse(html), project, context).body().html();}+ @Override+ public Node parse(String markdown) {+ MutableDataHolder options = setupOptions();+ Parser parser = Parser.builder(options).build();+ return parser.parse(markdown);+ }+}diff --git a/server-core/src/main/java/io/onedev/server/util/markdown/MarkdownManager.java b/server-core/src/main/java/io/onedev/server/util/markdown/MarkdownManager.javaindex e7671bc66f..7566e2a17f 100644--- a/server-core/src/main/java/io/onedev/server/util/markdown/MarkdownManager.java+++ b/server-core/src/main/java/io/onedev/server/util/markdown/MarkdownManager.java@@ -4,6 +4,8 @@ import javax.annotation.Nullable;import org.jsoup.nodes.Document;+import com.vladsch.flexmark.ast.Node;+import io.onedev.server.model.Project;public interface MarkdownManager {@@ -19,6 +21,8 @@ public interface MarkdownManager {*/String render(String markdown);+ Node parse(String markdown);+Document process(Document document, @Nullable Project project, @Nullable Object context);String process(String html, @Nullable Project project, @Nullable Object context);diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportOption.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportOption.javaindex 897c6cb4f4..134c367ad0 100644--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportOption.java+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportOption.java@@ -39,7 +39,7 @@ public class GitHubImportOption implements Serializable, Validatable {private String assigneesIssueField;- private List<IssueLabelMapping> labelMappings = new ArrayList<>();+ private List<IssueLabelMapping> issueLabelMappings = new ArrayList<>();@Editable(order=100, name="Repositories to Import")@Size(min=1, max=10000, message="No repositories to import")@@ -94,7 +94,7 @@ public class GitHubImportOption implements Serializable, Validatable {return choices;}- @Editable(order=350, description="Specify a multi-value issue custom field of type user to "+ @Editable(order=350, description="Specify a multi-value user field to "+ "hold assignees information")@ChoiceProvider("getAssigneesIssueFieldChoices")@NotEmpty@@ -116,13 +116,15 @@ public class GitHubImportOption implements Serializable, Validatable {return choices;}- @Editable(order=400, description="Map GitHub issue label to OneDev issue custom field of enum type")- public List<IssueLabelMapping> getLabelMappings() {- return labelMappings;+ @Editable(order=400, description="Specify how to map GitHub issue labels to OneDev custom "+ + "fields. Only multi-valued enum field can be used here. Unmapped labels will be "+ + "reflected in issue description")+ public List<IssueLabelMapping> getIssueLabelMappings() {+ return issueLabelMappings;}- public void setLabelMappings(List<IssueLabelMapping> labelMappings) {- this.labelMappings = labelMappings;+ public void setIssueLabelMappings(List<IssueLabelMapping> issueLabelMappings) {+ this.issueLabelMappings = issueLabelMappings;}@Overridediff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportSource.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportSource.javaindex cf4eba6f07..3827aad627 100644--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportSource.java+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportSource.java@@ -2,20 +2,38 @@ package io.onedev.server.plugin.imports.github;import java.io.Serializable;+import javax.validation.ConstraintValidatorContext;+import javax.ws.rs.client.Client;+import javax.ws.rs.client.ClientBuilder;+import javax.ws.rs.client.Invocation;+import javax.ws.rs.client.WebTarget;+import javax.ws.rs.core.Response;++import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;import org.hibernate.validator.constraints.NotEmpty;import io.onedev.commons.utils.StringUtils;+import io.onedev.server.util.JerseyUtils;+import io.onedev.server.util.validation.Validatable;+import io.onedev.server.util.validation.annotation.ClassValidating;import io.onedev.server.web.editable.annotation.Editable;@Editable-public class GitHubImportSource implements Serializable {+@ClassValidating+public class GitHubImportSource implements Serializable, Validatable {private static final long serialVersionUID = 1L;+ private static final String PROP_API_URL = "apiUrl";++ private static final String PROP_ACCESS_TOKEN = "accessToken";+private String apiUrl = "https://api.github.com";private String accessToken;+ private boolean prepopulateImportOptions = true;+@Editable(order=10, name="GitHub API URL", description="Specify GitHub API url, for instance <tt>https://api.github.com</tt>")@NotEmptypublic String getApiUrl() {@@ -37,8 +55,56 @@ public class GitHubImportSource implements Serializable {this.accessToken = accessToken;}+ @Editable(order=200, description="If checked, import options will be pre-populated based on all accessible "+ + "repositories and its settings")+ public boolean isPrepopulateImportOptions() {+ return prepopulateImportOptions;+ }++ public void setPrepopulateImportOptions(boolean prepopulateImportOptions) {+ this.prepopulateImportOptions = prepopulateImportOptions;+ }+public String getApiEndpoint(String apiPath) {return StringUtils.stripEnd(apiUrl, "/") + "/" + StringUtils.stripStart(apiPath, "/");}++ @Override+ public boolean isValid(ConstraintValidatorContext context) {+ Client client = ClientBuilder.newClient();+ client.register(HttpAuthenticationFeature.basic("git", getAccessToken()));+ try {+ WebTarget target = client.target(getApiEndpoint("/user"));+ Invocation.Builder builder = target.request();+ try (Response response = builder.get()) {+ if (response.getStatus() == 401) {+ context.disableDefaultConstraintViolation();+ String errorMessage = "Authentication failed";+ context.buildConstraintViolationWithTemplate(errorMessage)+ .addPropertyNode(PROP_ACCESS_TOKEN).addConstraintViolation();+ return false;+ } else {+ String errorMessage = JerseyUtils.checkStatus(response);+ if (errorMessage != null) {+ context.disableDefaultConstraintViolation();+ context.buildConstraintViolationWithTemplate(errorMessage)+ .addPropertyNode(PROP_API_URL).addConstraintViolation();+ return false;+ }+ }+ }+ } catch (Exception e) {+ context.disableDefaultConstraintViolation();+ String errorMessage = "Error connecting api service";+ if (e.getMessage() != null)+ errorMessage += ": " + e.getMessage();+ context.buildConstraintViolationWithTemplate(errorMessage)+ .addPropertyNode(PROP_API_URL).addConstraintViolation();+ return false;+ } finally {+ client.close();+ }+ return true;+ }}diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImporter.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImporter.javaindex 3fd1c7ab77..d265f323c9 100644--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImporter.java+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImporter.java@@ -6,6 +6,8 @@ import java.util.ArrayList;import java.util.Collection;import java.util.HashMap;import java.util.HashSet;+import java.util.LinkedHashMap;+import java.util.LinkedHashSet;import java.util.List;import java.util.Map;import java.util.Optional;@@ -21,6 +23,7 @@ import javax.ws.rs.core.Response;import org.apache.commons.lang3.StringUtils;import org.apache.http.client.utils.URIBuilder;+import org.glassfish.jersey.client.ClientProperties;import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;import org.joda.time.format.ISODateTimeFormat;import org.unbescape.html.HtmlEscape;@@ -29,12 +32,9 @@ import com.fasterxml.jackson.databind.JsonNode;import io.onedev.commons.utils.ExplicitException;import io.onedev.server.OneDev;-import io.onedev.server.entitymanager.IssueCommentManager;-import io.onedev.server.entitymanager.IssueFieldManager;import io.onedev.server.entitymanager.IssueManager;import io.onedev.server.entitymanager.MilestoneManager;import io.onedev.server.entitymanager.ProjectManager;-import io.onedev.server.entitymanager.RoleManager;import io.onedev.server.entitymanager.SettingManager;import io.onedev.server.entitymanager.UserManager;import io.onedev.server.model.Issue;@@ -43,13 +43,13 @@ import io.onedev.server.model.IssueField;import io.onedev.server.model.Milestone;import io.onedev.server.model.Project;import io.onedev.server.model.User;-import io.onedev.server.model.UserAuthorization;import io.onedev.server.model.support.LastUpdate;import io.onedev.server.model.support.administration.GlobalIssueSetting;import io.onedev.server.model.support.inputspec.InputSpec;import io.onedev.server.model.support.issue.field.spec.FieldSpec;-import io.onedev.server.security.SecurityUtils;+import io.onedev.server.persistence.dao.Dao;import io.onedev.server.storage.StorageManager;+import io.onedev.server.util.Pair;import io.onedev.server.util.SimpleLogger;import io.onedev.server.web.page.project.imports.ProjectImporter;@@ -69,20 +69,33 @@ public class GitHubImporter extends ProjectImporter<GitHubImportSource, GitHubIm@Overridepublic GitHubImportOption getImportOption(GitHubImportSource importSource, SimpleLogger logger) {GitHubImportOption importOption = new GitHubImportOption();- Client client = newClient(importSource);- try {- for (JsonNode repoNode: list(client, importSource, "/user/repos", logger)) {- String repoName = repoNode.get("name").asText();- String ownerName = repoNode.get("owner").get("login").asText();- GitHubImport aImport = new GitHubImport();- aImport.setGitHubRepo(ownerName + "/" + repoName);- aImport.setOneDevProject(ownerName + "-" + repoName);- importOption.getImports().add(aImport);+ if (importSource.isPrepopulateImportOptions()) {+ Client client = newClient(importSource);+ try {+ Set<String> labels = new LinkedHashSet<>();+ for (JsonNode repoNode: list(client, importSource, "/user/repos", logger)) {+ String repoName = repoNode.get("name").asText();+ String ownerName = repoNode.get("owner").get("login").asText();+ GitHubImport aImport = new GitHubImport();+ aImport.setGitHubRepo(ownerName + "/" + repoName);+ aImport.setOneDevProject(ownerName + "-" + repoName);+ importOption.getImports().add(aImport);++ for (JsonNode labelNode: list(client, importSource,+ "/repos/" + aImport.getGitHubRepo() + "/labels", logger)) {+ labels.add(labelNode.get("name").asText());+ }+ }+ for (String label: labels) {+ IssueLabelMapping mapping = new IssueLabelMapping();+ mapping.setGitHubIssueLabel(label);+ importOption.getIssueLabelMappings().add(mapping);+ }+ } finally {+ client.close();}- return importOption;- } finally {- client.close();}+ return importOption;}@Nullable@@ -107,12 +120,24 @@ public class GitHubImporter extends ProjectImporter<GitHubImportSource, GitHubImStorageManager storageManager = OneDev.getInstance(StorageManager.class);Client client = newClient(importSource);- Set<String> loginsWithoutAccount = new HashSet<>();+ Set<String> nonExistentLogins = new HashSet<>();Set<String> unmappedIssueLabels = new HashSet<>();+ Map<String, Pair<FieldSpec, String>> labelMappings = new HashMap<>();++ for (IssueLabelMapping mapping: importOption.getIssueLabelMappings()) {+ String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");+ String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");+ FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);+ if (fieldSpec == null)+ throw new ExplicitException("No field spec found: " + oneDevFieldName);+ labelMappings.put(mapping.getGitHubIssueLabel(), new Pair<>(fieldSpec, oneDevFieldValue));+ }+Collection<Long> projectIds = new ArrayList<>();try {Map<String, Optional<User>> users = new HashMap<>();+ boolean importIssues = false;for (GitHubImport aImport: importOption.getImports()) {logger.log("Cloning code of repository " + aImport.getGitHubRepo() + "...");JsonNode repoNode = get(client, importSource, "/repos/" + aImport.getGitHubRepo(), logger);@@ -123,6 +148,7 @@ public class GitHubImporter extends ProjectImporter<GitHubImportSource, GitHubImboolean isPrivate = repoNode.get("private").asBoolean();if (!isPrivate && importOption.getPublicRole() != null)project.setDefaultRole(importOption.getPublicRole());+URIBuilder builder = new URIBuilder(repoNode.get("clone_url").asText());if (isPrivate)builder.setUserInfo("git", importSource.getAccessToken());@@ -130,17 +156,10 @@ public class GitHubImporter extends ProjectImporter<GitHubImportSource, GitHubImif (!dryRun) {projectManager.clone(project, builder.build().toString());projectIds.add(project.getId());- } else {- User user = SecurityUtils.getUser();- UserAuthorization authorization = new UserAuthorization();- authorization.setProject(project);- authorization.setUser(user);- authorization.setRole(OneDev.getInstance(RoleManager.class).getOwner());- project.getUserAuthorizations().add(authorization);- user.getAuthorizations().add(authorization);}if (aImport.isImportIssues()) {+ importIssues = true;Map<Long, Milestone> milestones = new HashMap<>();logger.log("Importing milestones of repository " + aImport.getGitHubRepo() + "...");String apiEndpoint = "/repos/" + aImport.getGitHubRepo() + "/milestones?state=all";@@ -166,6 +185,68 @@ public class GitHubImporter extends ProjectImporter<GitHubImportSource, GitHubImAtomicInteger numOfImportedIssues = new AtomicInteger(0);PageDataConsumer pageDataConsumer = new PageDataConsumer() {+ /*+ @Nullable+ private String processAttachments(String issueUUID, Long issueNumber, @Nullable String markdown,+ Set<String> tooLargeAttachments) {+ if (markdown == null)+ return markdown;++ Map<String, String> attachments = new HashMap<>();++ Node node = OneDev.getInstance(MarkdownManager.class).parse(markdown);+ new NodeVisitor(new VisitHandler<?>[] {}) {++ @Override+ public void visit(Node node) {+ super.visit(node);+ if (node instanceof InlineLinkNode) {+ InlineLinkNode link = (InlineLinkNode)node;+ String url = link.getUrl().toString();+ if (url.startsWith(repoUrl + "/files/")) {+ attachments.put(url, link.getText().toString());+ }+ }+ }++ }.visit(node);++ for (Map.Entry<String, String> entry: attachments.entrySet()) {+ String attachmentUrl = entry.getKey();+ String attachmentName = entry.getValue();++ WebTarget target = client.target(attachmentUrl);+ Invocation.Builder builder = target.request();+ try (Response response = builder.get()) {+ String errorMessage = JerseyUtils.checkStatus(response);+ if (errorMessage != null) {+ throw new ExplicitException(String.format(+ "Error downloading attachment (url: %s, error message: %s)",+ attachmentUrl, errorMessage));+ }+ try (InputStream is = response.readEntity(InputStream.class)) {+ String oneDevAttachmentName = project.saveAttachment(issueUUID, attachmentName, is);+ String oneDevAttachmentUrl = project.getAttachmentUrlPath(issueUUID, oneDevAttachmentName);+ markdown = markdown.replace("(" + attachmentName + ")", "(" + oneDevAttachmentUrl + ")");+ } catch (AttachmentTooLargeException e) {+ tooLargeAttachments.add(aImport.getGitHubRepo() + "#" + issueNumber + ":" + attachmentName);+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ }+ }++ return markdown;+ }+ */++ private String joinAsMultilineHtml(List<String> values) {+ List<String> escapedValues = new ArrayList<>();+ for (String value: values)+ escapedValues.add(HtmlEscape.escapeHtml5(value));+ return StringUtils.join(escapedValues, "<br>");+ }+@Overridepublic void consume(List<JsonNode> pageData) throws InterruptedException {for (JsonNode issueNode: pageData) {@@ -193,7 +274,7 @@ public class GitHubImporter extends ProjectImporter<GitHubImportSource, GitHubImissue.setSubmitter(user);} else {issue.setSubmitterName(login);- loginsWithoutAccount.add(login);+ nonExistentLogins.add(login);}issue.setSubmitDate(ISODateTimeFormat.dateTimeNoMillis()@@ -207,8 +288,7 @@ public class GitHubImporter extends ProjectImporter<GitHubImportSource, GitHubImlastUpdate.setUserName(issue.getSubmitterName());issue.setLastUpdate(lastUpdate);- if (!dryRun)- OneDev.getInstance(IssueManager.class).save(issue);+ Map<String, String> extraIssueInfo = new LinkedHashMap<>();for (JsonNode assigneeNode: issueNode.get("assignees")) {IssueField assigneeField = new IssueField();@@ -222,10 +302,9 @@ public class GitHubImporter extends ProjectImporter<GitHubImportSource, GitHubImassigneeField.setValue(user.getName());} else {assigneeField.setValue(login);- loginsWithoutAccount.add(login);+ nonExistentLogins.add(login);}- if (!dryRun)- getIssueFieldManager().save(assigneeField);+ issue.getFields().add(assigneeField);}String apiEndpoint = "/repos/" + aImport.getGitHubRepo()@@ -244,39 +323,61 @@ public class GitHubImporter extends ProjectImporter<GitHubImportSource, GitHubImcomment.setUser(user);} else {comment.setUserName(login);- loginsWithoutAccount.add(login);+ nonExistentLogins.add(login);}-- if (!dryRun)- OneDev.getInstance(IssueCommentManager.class).save(comment);++ issue.getComments().add(comment);}+ issue.setCommentCount(issue.getComments().size());+apiEndpoint = "/repos/" + aImport.getGitHubRepo()+ "/issues/" + issue.getNumber() + "/labels";+ List<String> currentUnmappedLabels = new ArrayList<>();for (JsonNode labelNode: list(client, importSource, apiEndpoint, logger)) {String labelName = labelNode.get("name").asText();- IssueField labelField = null;- for (IssueLabelMapping mapping: importOption.getLabelMappings()) {- if (mapping.getGitHubIssueLabel().equals(labelName)) {- String fieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");- FieldSpec fieldSpec = getIssueSetting().getFieldSpecMap(null).get(fieldName);- if (fieldSpec != null) {- labelField = new IssueField();- labelField.setIssue(issue);- String fieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");- labelField.setName(fieldName);- labelField.setType(InputSpec.ENUMERATION);- labelField.setValue(fieldValue);- labelField.setOrdinal(fieldSpec.getOrdinal(fieldValue));- if (!dryRun)- getIssueFieldManager().save(labelField);- }- break;- }+ Pair<FieldSpec, String> mapped = labelMappings.get(labelName);+ if (mapped != null) {+ IssueField tagField = new IssueField();+ tagField.setIssue(issue);+ tagField.setName(mapped.getFirst().getName());+ tagField.setType(InputSpec.ENUMERATION);+ tagField.setValue(mapped.getSecond());+ tagField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));+ issue.getFields().add(tagField);+ } else {+ currentUnmappedLabels.add(HtmlEscape.escapeHtml5(labelName));+ unmappedIssueLabels.add(HtmlEscape.escapeHtml5(labelName));}- if (labelField == null)- unmappedIssueLabels.add(labelName);}++ if (!currentUnmappedLabels.isEmpty())+ extraIssueInfo.put("Labels", joinAsMultilineHtml(currentUnmappedLabels));++ if (!extraIssueInfo.isEmpty()) {+ StringBuilder builder = new StringBuilder("|");+ for (String key: extraIssueInfo.keySet())+ builder.append(key).append("|");+ builder.append("\n|");+ extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));+ builder.append("\n|");+ for (String value: extraIssueInfo.values())+ builder.append(value).append("|");++ if (issue.getDescription() != null)+ issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());+ else+ issue.setDescription(builder.toString());+ }++ if (!dryRun) {+ OneDev.getInstance(IssueManager.class).save(issue);+ for (IssueField field: issue.getFields())+ OneDev.getInstance(Dao.class).persist(field);+ for (IssueComment comment: issue.getComments())+ OneDev.getInstance(Dao.class).persist(comment);+ }+if (numOfImportedIssues.incrementAndGet() % 25 == 0)logger.log("Imported " + numOfImportedIssues.get() + " issues");}@@ -287,17 +388,29 @@ public class GitHubImporter extends ProjectImporter<GitHubImportSource, GitHubImlist(client, importSource, apiEndpoint, pageDataConsumer, logger);}}- List<String> result = new ArrayList<>();- result.add("All repositories imported successfully");- if (!loginsWithoutAccount.isEmpty()) {- result.add("<b>NOTE:</b> GitHub logins without public email or public email can not be mapped to OneDev account: "- + HtmlEscape.escapeHtml5(loginsWithoutAccount.toString()));- }++ StringBuilder feedback = new StringBuilder("Repositories imported successfully");++ boolean hasNotes = !unmappedIssueLabels.isEmpty() || !nonExistentLogins.isEmpty() || importIssues;++ if (hasNotes)+ feedback.append("<br><br><b>NOTE:</b><ul>");+if (!unmappedIssueLabels.isEmpty()) {- result.add("<b>NOTE:</b> Issue labels not mapped to custom field: "+ feedback.append("<li> GitHub issue labels not mapped to OneDev custom field (mentioned as extra info in issue description): "+ HtmlEscape.escapeHtml5(unmappedIssueLabels.toString()));}- return StringUtils.join(result, "<br>");+ if (!nonExistentLogins.isEmpty()) {+ feedback.append("<li> GitHub logins without public email or public email can not be mapped to OneDev account: "+ + HtmlEscape.escapeHtml5(nonExistentLogins.toString()));+ }+ if (importIssues)+ feedback.append("<li> Attachments in issues and comments are not imported due to GitHub limitation");++ if (hasNotes)+ feedback.append("</ul>");++ return feedback.toString();} catch (Exception e) {for (Long projectId: projectIds)storageManager.deleteProjectDir(projectId);@@ -311,10 +424,6 @@ public class GitHubImporter extends ProjectImporter<GitHubImportSource, GitHubImreturn OneDev.getInstance(SettingManager.class).getIssueSetting();}- private IssueFieldManager getIssueFieldManager() {- return OneDev.getInstance(IssueFieldManager.class);- }-private List<JsonNode> list(Client client, GitHubImportSource importSource,String apiPath, SimpleLogger logger) {List<JsonNode> result = new ArrayList<>();@@ -384,8 +493,7 @@ public class GitHubImporter extends ProjectImporter<GitHubImportSource, GitHubImstatus, errorMessage));}} else {- throw new RuntimeException("Http request failed with status " + status- + ", check server log for detaiils");+ throw new ExplicitException(String.format("Http request failed (status: %s)", status));}}return response.readEntity(JsonNode.class);@@ -395,6 +503,7 @@ public class GitHubImporter extends ProjectImporter<GitHubImportSource, GitHubImprivate Client newClient(GitHubImportSource importSource) {Client client = ClientBuilder.newClient();+ client.property(ClientProperties.FOLLOW_REDIRECTS, true);client.register(HttpAuthenticationFeature.basic("git", importSource.getAccessToken()));return client;}diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportOption.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportOption.javaindex f3ce34d6ac..b03e6e16a9 100644--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportOption.java+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportOption.java@@ -60,7 +60,8 @@ public class YouTrackImportOption implements Serializable, Validatable {}@Editable(order=300, description="Specify how to map YouTrack issue tags to OneDev issue custom "- + "fields. Unmapped tags will be reflected in issue description")+ + "fields. Only multi-valued enum field can be used here. Unmapped tags will be "+ + "reflected in issue description")public List<IssueTagMapping> getIssueTagMappings() {return issueTagMappings;}diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportSource.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportSource.javaindex ef06b754fd..e7d052383e 100644--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportSource.java+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportSource.java@@ -15,6 +15,7 @@ import org.hibernate.validator.constraints.NotEmpty;import com.fasterxml.jackson.databind.JsonNode;import io.onedev.commons.utils.StringUtils;+import io.onedev.server.util.JerseyUtils;import io.onedev.server.util.validation.Validatable;import io.onedev.server.util.validation.annotation.ClassValidating;import io.onedev.server.web.editable.annotation.Editable;@@ -77,8 +78,7 @@ public class YouTrackImportSource implements Serializable, Validatable {}@Editable(order=400, description="If checked, import options will be pre-populated based on all accessible "- + "projects and its settings. In case there are too many projects and settings, you may want to "- + "uncheck this and provide import option manually")+ + "projects and its settings")public boolean isPrepopulateImportOptions() {return prepopulateImportOptions;}@@ -99,7 +99,7 @@ public class YouTrackImportSource implements Serializable, Validatable {WebTarget target = client.target(getApiEndpoint("/users/me?fields=guest"));Invocation.Builder builder = target.request();try (Response response = builder.get()) {- String errorMessage = YouTrackImporter.checkStatus(response);+ String errorMessage = JerseyUtils.checkStatus(response);if (errorMessage != null) {context.disableDefaultConstraintViolation();context.buildConstraintViolationWithTemplate(errorMessage)diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImporter.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImporter.javaindex 8c54deafb1..9f3237b82d 100644--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImporter.java+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImporter.java@@ -58,6 +58,7 @@ import io.onedev.server.persistence.dao.Dao;import io.onedev.server.security.SecurityUtils;import io.onedev.server.storage.StorageManager;import io.onedev.server.util.DateUtils;+import io.onedev.server.util.JerseyUtils;import io.onedev.server.util.Pair;import io.onedev.server.util.SimpleLogger;import io.onedev.server.web.page.project.imports.ProjectImporter;@@ -182,7 +183,7 @@ public class YouTrackImporter extends ProjectImporter<YouTrackImportSource, YouTprojectDescriptions.put(projectName, projectNode.get("description").asText(null));}- Set<String> unmappedAccounts = new LinkedHashSet<>();+ Set<String> nonExistentLogins = new LinkedHashSet<>();Set<String> unmappedIssueTags = new LinkedHashSet<>();Set<String> unmappedIssueFields = new LinkedHashSet<>();Set<String> unmappedIssueStates = new LinkedHashSet<>();@@ -241,32 +242,28 @@ public class YouTrackImporter extends ProjectImporter<YouTrackImportSource, YouTif (!attachmentUrl.startsWith("/api"))throw new ExplicitException("Unexpected attachment url: " + attachmentUrl);attachmentUrl = attachmentUrl.substring("/api".length());-- WebTarget target = client.target(importSource.getApiEndpoint(attachmentUrl));++ String endpoint = importSource.getApiEndpoint(attachmentUrl);+ WebTarget target = client.target(endpoint);Invocation.Builder builder = target.request();try (Response response = builder.get()) {- int status = response.getStatus();- if (status != 200) {- String errorMessage = response.readEntity(String.class);- if (StringUtils.isNotBlank(errorMessage)) {- throw new ExplicitException("Error downloading attachment: " + errorMessage);- } else {- throw new RuntimeException("Attachment download failed with status " + status- + ", check server log for detaiils");- }- } else {- try (InputStream is = response.readEntity(InputStream.class)) {- String oneDevAttachmentName = project.saveAttachment(issueUUID, attachmentName, is);- String oneDevAttachmentUrl = project.getAttachmentUrlPath(issueUUID, oneDevAttachmentName);- if (markdown.contains("(" + attachmentName + ")")) {- markdown = markdown.replace("(" + attachmentName + ")", "(" + oneDevAttachmentUrl + ")");- } else {- unreferencedAttachments.put(attachmentName, oneDevAttachmentUrl);- }- } catch (IOException e) {- throw new RuntimeException(e);- }+ String errorMessage = JerseyUtils.checkStatus(response);+ if (errorMessage != null) {+ throw new ExplicitException(String.format(+ "Error downloading attachment (url: %s, error message: %s)",+ endpoint, errorMessage));}+ try (InputStream is = response.readEntity(InputStream.class)) {+ String oneDevAttachmentName = project.saveAttachment(issueUUID, attachmentName, is);+ String oneDevAttachmentUrl = project.getAttachmentUrlPath(issueUUID, oneDevAttachmentName);+ if (markdown.contains("(" + attachmentName + ")")) {+ markdown = markdown.replace("(" + attachmentName + ")", "(" + oneDevAttachmentUrl + ")");+ } else {+ unreferencedAttachments.put(attachmentName, oneDevAttachmentUrl);+ }+ } catch (IOException e) {+ throw new RuntimeException(e);+ }}}}@@ -338,11 +335,11 @@ public class YouTrackImporter extends ProjectImporter<YouTrackImportSource, YouTissue.setSubmitter(user);} else {issue.setSubmitterName(fullName);- unmappedAccounts.add(login);+ nonExistentLogins.add(login);}} else {issue.setSubmitterName(fullName);- unmappedAccounts.add(login);+ nonExistentLogins.add(login);}} else {issue.setSubmitter(SecurityUtils.getUser());@@ -491,11 +488,11 @@ public class YouTrackImporter extends ProjectImporter<YouTrackImportSource, YouTfieldValue = user.getName();} else {fieldValue = fullName;- unmappedAccounts.add(login);+ nonExistentLogins.add(login);}} else {fieldValue = fullName;- unmappedAccounts.add(login);+ nonExistentLogins.add(login);}IssueField issueField = new IssueField();@@ -531,11 +528,11 @@ public class YouTrackImporter extends ProjectImporter<YouTrackImportSource, YouTfieldValue = user.getName();} else {fieldValue = fullName;- unmappedAccounts.add(login);+ nonExistentLogins.add(login);}} else {fieldValue = fullName;- unmappedAccounts.add(login);+ nonExistentLogins.add(login);}IssueField issueField = new IssueField();@@ -763,11 +760,11 @@ public class YouTrackImporter extends ProjectImporter<YouTrackImportSource, YouTcomment.setUser(user);} else {comment.setUserName(fullName);- unmappedAccounts.add(login);+ nonExistentLogins.add(login);}} else {comment.setUserName(fullName);- unmappedAccounts.add(login);+ nonExistentLogins.add(login);}} else {issue.setSubmitter(SecurityUtils.getUser());@@ -817,7 +814,7 @@ public class YouTrackImporter extends ProjectImporter<YouTrackImportSource, YouT|| !unmappedIssueFields.isEmpty()|| !mismatchedIssueFields.isEmpty()|| !unmappedIssueTags.isEmpty()- || !unmappedAccounts.isEmpty()+ || !nonExistentLogins.isEmpty()|| !tooLargeAttachments.isEmpty();if (hasNotes)@@ -846,9 +843,9 @@ public class YouTrackImporter extends ProjectImporter<YouTrackImportSource, YouTfeedback.append("<li> YouTrack issue tags not mapped to OneDev custom field (mentioned as extra info in issue description): "+ HtmlEscape.escapeHtml5(unmappedIssueTags.toString()));}- if (!unmappedAccounts.isEmpty()) {+ if (!nonExistentLogins.isEmpty()) {feedback.append("<li> YouTrack logins without email or email can not be mapped to OneDev account: "- + HtmlEscape.escapeHtml5(unmappedAccounts.toString()));+ + HtmlEscape.escapeHtml5(nonExistentLogins.toString()));}if (!tooLargeAttachments.isEmpty()) {feedback.append("<li> Too large attachments: "@@ -918,7 +915,7 @@ public class YouTrackImporter extends ProjectImporter<YouTrackImportSource, YouTWebTarget target = client.target(apiEndpoint);Invocation.Builder builder = target.request();try (Response response = builder.get()) {- String errorMessage = checkStatus(response);+ String errorMessage = JerseyUtils.checkStatus(response);if (errorMessage != null)throw new ExplicitException(errorMessage);else@@ -926,22 +923,6 @@ public class YouTrackImporter extends ProjectImporter<YouTrackImportSource, YouT}}- @Nullable- public static String checkStatus(Response response) {- int status = response.getStatus();- if (status != 200) {- String errorMessage = response.readEntity(String.class);- if (StringUtils.isNotBlank(errorMessage)) {- return String.format("Http request failed (status code: %d, error message: %s)",- status, errorMessage);- } else {- return String.format("Http request failed (status code: %d)", status);- }- } else {- return null;- }- }-private Client newClient(YouTrackImportSource importSource) {Client client = ClientBuilder.newClient();client.register(HttpAuthenticationFeature.basic(importSource.getUserName(), importSource.getPassword()));