Skip to content

The Unified and Autonomous Development Platform

Improve GitHub importer

Robin Shenfc7669f73f9d
diff --git a/server-core/src/main/java/io/onedev/server/model/Project.java b/server-core/src/main/java/io/onedev/server/model/Project.java
index 96992f3775..5e792d5f11 100644
--- a/server-core/src/main/java/io/onedev/server/model/Project.java
+++ b/server-core/src/main/java/io/onedev/server/model/Project.java
@@ -113,6 +113,7 @@ import io.onedev.server.persistence.TransactionManager;
import io.onedev.server.security.SecurityUtils;
import io.onedev.server.storage.AttachmentStorageManager;
import io.onedev.server.storage.StorageManager;
+import io.onedev.server.util.AttachmentTooLargeException;
import io.onedev.server.util.CollectionUtils;
import io.onedev.server.util.ComponentContext;
import io.onedev.server.util.NameAware;
@@ -1521,7 +1522,7 @@ public class Project extends AbstractEntity implements NameAware {
while (-1 != (n = attachmentStream.read(buffer))) {
count += n;
if (count > MAX_ATTACHMENT_SIZE) {
- throw new RuntimeException("Upload must be less than "
+ throw new AttachmentTooLargeException("Upload must be less than "
+ FileUtils.byteCountToDisplaySize(MAX_ATTACHMENT_SIZE));
}
os.write(buffer, 0, n);
diff --git a/server-core/src/main/java/io/onedev/server/util/AttachmentTooLargeException.java b/server-core/src/main/java/io/onedev/server/util/AttachmentTooLargeException.java
new file mode 100644
index 0000000000..a6a8e47db6
--- /dev/null
+++ b/server-core/src/main/java/io/onedev/server/util/AttachmentTooLargeException.java
@@ -0,0 +1,13 @@
+package io.onedev.server.util;
+
+import io.onedev.commons.utils.ExplicitException;
+
+public class AttachmentTooLargeException extends ExplicitException {
+
+ private static final long serialVersionUID = 1L;
+
+ public AttachmentTooLargeException(String message) {
+ super(message);
+ }
+
+}
diff --git a/server-core/src/main/java/io/onedev/server/util/JerseyUtils.java b/server-core/src/main/java/io/onedev/server/util/JerseyUtils.java
new file mode 100644
index 0000000000..2fdcd7d008
--- /dev/null
+++ b/server-core/src/main/java/io/onedev/server/util/JerseyUtils.java
@@ -0,0 +1,26 @@
+package io.onedev.server.util;
+
+import javax.annotation.Nullable;
+import javax.ws.rs.core.Response;
+
+import io.onedev.commons.utils.StringUtils;
+
+public class JerseyUtils {
+
+ @Nullable
+ public static String checkStatus(Response response) {
+ int status = response.getStatus();
+ if (status != 200) {
+ String errorMessage = response.readEntity(String.class);
+ if (StringUtils.isNotBlank(errorMessage)) {
+ return String.format("Http request failed (status code: %d, error message: %s)",
+ status, errorMessage);
+ } else {
+ return String.format("Http request failed (status code: %d)", status);
+ }
+ } else {
+ return null;
+ }
+ }
+
+}
diff --git a/server-core/src/main/java/io/onedev/server/util/markdown/DefaultMarkdownManager.java b/server-core/src/main/java/io/onedev/server/util/markdown/DefaultMarkdownManager.java
index ee3bdb2ba3..379208446b 100644
--- a/server-core/src/main/java/io/onedev/server/util/markdown/DefaultMarkdownManager.java
+++ b/server-core/src/main/java/io/onedev/server/util/markdown/DefaultMarkdownManager.java
@@ -42,8 +42,7 @@ public class DefaultMarkdownManager implements MarkdownManager {
this.htmlTransformers = htmlTransformers;
}
- @Override
- public String render(String markdown) {
+ private MutableDataHolder setupOptions() {
List<Extension> extensions = new ArrayList<>();
extensions.add(AnchorLinkExtension.create());
extensions.add(TablesExtension.create());
@@ -53,7 +52,7 @@ public class DefaultMarkdownManager implements MarkdownManager {
extensions.add(AutolinkExtension.create());
extensions.addAll(contributedExtensions);
- MutableDataHolder options = new MutableDataSet()
+ return new MutableDataSet()
.set(HtmlRenderer.GENERATE_HEADER_ID, true)
.set(AnchorLinkExtension.ANCHORLINKS_SET_NAME, true)
.set(AnchorLinkExtension.ANCHORLINKS_WRAP_TEXT, false)
@@ -65,13 +64,13 @@ public class DefaultMarkdownManager implements MarkdownManager {
.set(TablesExtension.DISCARD_EXTRA_COLUMNS, true)
.set(TablesExtension.HEADER_SEPARATOR_COLUMN_MATCH, true)
.set(Parser.EXTENSIONS, extensions);
-
- Parser parser = Parser.builder(options).build();
-
- HtmlRenderer htmlRenderer = HtmlRenderer.builder(options).build();
-
- Node document = parser.parse(markdown);
- return htmlRenderer.render(document);
+ }
+
+ @Override
+ public String render(String markdown) {
+ MutableDataHolder options = setupOptions();
+ Node node = parse(markdown);
+ return HtmlRenderer.builder(options).build().render(node);
}
@Override
@@ -94,4 +93,11 @@ public class DefaultMarkdownManager implements MarkdownManager {
return process(HtmlUtils.parse(html), project, context).body().html();
}
+ @Override
+ public Node parse(String markdown) {
+ MutableDataHolder options = setupOptions();
+ Parser parser = Parser.builder(options).build();
+ return parser.parse(markdown);
+ }
+
}
diff --git a/server-core/src/main/java/io/onedev/server/util/markdown/MarkdownManager.java b/server-core/src/main/java/io/onedev/server/util/markdown/MarkdownManager.java
index e7671bc66f..7566e2a17f 100644
--- a/server-core/src/main/java/io/onedev/server/util/markdown/MarkdownManager.java
+++ b/server-core/src/main/java/io/onedev/server/util/markdown/MarkdownManager.java
@@ -4,6 +4,8 @@ import javax.annotation.Nullable;
import org.jsoup.nodes.Document;
+import com.vladsch.flexmark.ast.Node;
+
import io.onedev.server.model.Project;
public interface MarkdownManager {
@@ -19,6 +21,8 @@ public interface MarkdownManager {
*/
String render(String markdown);
+ Node parse(String markdown);
+
Document process(Document document, @Nullable Project project, @Nullable Object context);
String process(String html, @Nullable Project project, @Nullable Object context);
diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportOption.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportOption.java
index 897c6cb4f4..134c367ad0 100644
--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportOption.java
+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportOption.java
@@ -39,7 +39,7 @@ public class GitHubImportOption implements Serializable, Validatable {
private String assigneesIssueField;
- private List<IssueLabelMapping> labelMappings = new ArrayList<>();
+ private List<IssueLabelMapping> issueLabelMappings = new ArrayList<>();
@Editable(order=100, name="Repositories to Import")
@Size(min=1, max=10000, message="No repositories to import")
@@ -94,7 +94,7 @@ public class GitHubImportOption implements Serializable, Validatable {
return choices;
}
- @Editable(order=350, description="Specify a multi-value issue custom field of type user to "
+ @Editable(order=350, description="Specify a multi-value user field to "
+ "hold assignees information")
@ChoiceProvider("getAssigneesIssueFieldChoices")
@NotEmpty
@@ -116,13 +116,15 @@ public class GitHubImportOption implements Serializable, Validatable {
return choices;
}
- @Editable(order=400, description="Map GitHub issue label to OneDev issue custom field of enum type")
- public List<IssueLabelMapping> getLabelMappings() {
- return labelMappings;
+ @Editable(order=400, description="Specify how to map GitHub issue labels to OneDev custom "
+ + "fields. Only multi-valued enum field can be used here. Unmapped labels will be "
+ + "reflected in issue description")
+ public List<IssueLabelMapping> getIssueLabelMappings() {
+ return issueLabelMappings;
}
- public void setLabelMappings(List<IssueLabelMapping> labelMappings) {
- this.labelMappings = labelMappings;
+ public void setIssueLabelMappings(List<IssueLabelMapping> issueLabelMappings) {
+ this.issueLabelMappings = issueLabelMappings;
}
@Override
diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportSource.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportSource.java
index cf4eba6f07..3827aad627 100644
--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportSource.java
+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImportSource.java
@@ -2,20 +2,38 @@ package io.onedev.server.plugin.imports.github;
import java.io.Serializable;
+import javax.validation.ConstraintValidatorContext;
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.ClientBuilder;
+import javax.ws.rs.client.Invocation;
+import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.Response;
+
+import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
import org.hibernate.validator.constraints.NotEmpty;
import io.onedev.commons.utils.StringUtils;
+import io.onedev.server.util.JerseyUtils;
+import io.onedev.server.util.validation.Validatable;
+import io.onedev.server.util.validation.annotation.ClassValidating;
import io.onedev.server.web.editable.annotation.Editable;
@Editable
-public class GitHubImportSource implements Serializable {
+@ClassValidating
+public class GitHubImportSource implements Serializable, Validatable {
private static final long serialVersionUID = 1L;
+ private static final String PROP_API_URL = "apiUrl";
+
+ private static final String PROP_ACCESS_TOKEN = "accessToken";
+
private String apiUrl = "https://api.github.com";
private String accessToken;
+ private boolean prepopulateImportOptions = true;
+
@Editable(order=10, name="GitHub API URL", description="Specify GitHub API url, for instance <tt>https://api.github.com</tt>")
@NotEmpty
public String getApiUrl() {
@@ -37,8 +55,56 @@ public class GitHubImportSource implements Serializable {
this.accessToken = accessToken;
}
+ @Editable(order=200, description="If checked, import options will be pre-populated based on all accessible "
+ + "repositories and its settings")
+ public boolean isPrepopulateImportOptions() {
+ return prepopulateImportOptions;
+ }
+
+ public void setPrepopulateImportOptions(boolean prepopulateImportOptions) {
+ this.prepopulateImportOptions = prepopulateImportOptions;
+ }
+
public String getApiEndpoint(String apiPath) {
return StringUtils.stripEnd(apiUrl, "/") + "/" + StringUtils.stripStart(apiPath, "/");
}
+
+ @Override
+ public boolean isValid(ConstraintValidatorContext context) {
+ Client client = ClientBuilder.newClient();
+ client.register(HttpAuthenticationFeature.basic("git", getAccessToken()));
+ try {
+ WebTarget target = client.target(getApiEndpoint("/user"));
+ Invocation.Builder builder = target.request();
+ try (Response response = builder.get()) {
+ if (response.getStatus() == 401) {
+ context.disableDefaultConstraintViolation();
+ String errorMessage = "Authentication failed";
+ context.buildConstraintViolationWithTemplate(errorMessage)
+ .addPropertyNode(PROP_ACCESS_TOKEN).addConstraintViolation();
+ return false;
+ } else {
+ String errorMessage = JerseyUtils.checkStatus(response);
+ if (errorMessage != null) {
+ context.disableDefaultConstraintViolation();
+ context.buildConstraintViolationWithTemplate(errorMessage)
+ .addPropertyNode(PROP_API_URL).addConstraintViolation();
+ return false;
+ }
+ }
+ }
+ } catch (Exception e) {
+ context.disableDefaultConstraintViolation();
+ String errorMessage = "Error connecting api service";
+ if (e.getMessage() != null)
+ errorMessage += ": " + e.getMessage();
+ context.buildConstraintViolationWithTemplate(errorMessage)
+ .addPropertyNode(PROP_API_URL).addConstraintViolation();
+ return false;
+ } finally {
+ client.close();
+ }
+ return true;
+ }
}
diff --git a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImporter.java b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImporter.java
index 3fd1c7ab77..d265f323c9 100644
--- a/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImporter.java
+++ b/server-plugin/server-plugin-import-github/src/main/java/io/onedev/server/plugin/imports/github/GitHubImporter.java
@@ -6,6 +6,8 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -21,6 +23,7 @@ import javax.ws.rs.core.Response;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.client.utils.URIBuilder;
+import org.glassfish.jersey.client.ClientProperties;
import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
import org.joda.time.format.ISODateTimeFormat;
import org.unbescape.html.HtmlEscape;
@@ -29,12 +32,9 @@ import com.fasterxml.jackson.databind.JsonNode;
import io.onedev.commons.utils.ExplicitException;
import io.onedev.server.OneDev;
-import io.onedev.server.entitymanager.IssueCommentManager;
-import io.onedev.server.entitymanager.IssueFieldManager;
import io.onedev.server.entitymanager.IssueManager;
import io.onedev.server.entitymanager.MilestoneManager;
import io.onedev.server.entitymanager.ProjectManager;
-import io.onedev.server.entitymanager.RoleManager;
import io.onedev.server.entitymanager.SettingManager;
import io.onedev.server.entitymanager.UserManager;
import io.onedev.server.model.Issue;
@@ -43,13 +43,13 @@ import io.onedev.server.model.IssueField;
import io.onedev.server.model.Milestone;
import io.onedev.server.model.Project;
import io.onedev.server.model.User;
-import io.onedev.server.model.UserAuthorization;
import io.onedev.server.model.support.LastUpdate;
import io.onedev.server.model.support.administration.GlobalIssueSetting;
import io.onedev.server.model.support.inputspec.InputSpec;
import io.onedev.server.model.support.issue.field.spec.FieldSpec;
-import io.onedev.server.security.SecurityUtils;
+import io.onedev.server.persistence.dao.Dao;
import io.onedev.server.storage.StorageManager;
+import io.onedev.server.util.Pair;
import io.onedev.server.util.SimpleLogger;
import io.onedev.server.web.page.project.imports.ProjectImporter;
@@ -69,20 +69,33 @@ public class GitHubImporter extends ProjectImporter<GitHubImportSource, GitHubIm
@Override
public GitHubImportOption getImportOption(GitHubImportSource importSource, SimpleLogger logger) {
GitHubImportOption importOption = new GitHubImportOption();
- Client client = newClient(importSource);
- try {
- for (JsonNode repoNode: list(client, importSource, "/user/repos", logger)) {
- String repoName = repoNode.get("name").asText();
- String ownerName = repoNode.get("owner").get("login").asText();
- GitHubImport aImport = new GitHubImport();
- aImport.setGitHubRepo(ownerName + "/" + repoName);
- aImport.setOneDevProject(ownerName + "-" + repoName);
- importOption.getImports().add(aImport);
+ if (importSource.isPrepopulateImportOptions()) {
+ Client client = newClient(importSource);
+ try {
+ Set<String> labels = new LinkedHashSet<>();
+ for (JsonNode repoNode: list(client, importSource, "/user/repos", logger)) {
+ String repoName = repoNode.get("name").asText();
+ String ownerName = repoNode.get("owner").get("login").asText();
+ GitHubImport aImport = new GitHubImport();
+ aImport.setGitHubRepo(ownerName + "/" + repoName);
+ aImport.setOneDevProject(ownerName + "-" + repoName);
+ importOption.getImports().add(aImport);
+
+ for (JsonNode labelNode: list(client, importSource,
+ "/repos/" + aImport.getGitHubRepo() + "/labels", logger)) {
+ labels.add(labelNode.get("name").asText());
+ }
+ }
+ for (String label: labels) {
+ IssueLabelMapping mapping = new IssueLabelMapping();
+ mapping.setGitHubIssueLabel(label);
+ importOption.getIssueLabelMappings().add(mapping);
+ }
+ } finally {
+ client.close();
}
- return importOption;
- } finally {
- client.close();
}
+ return importOption;
}
@Nullable
@@ -107,12 +120,24 @@ public class GitHubImporter extends ProjectImporter<GitHubImportSource, GitHubIm
StorageManager storageManager = OneDev.getInstance(StorageManager.class);
Client client = newClient(importSource);
- Set<String> loginsWithoutAccount = new HashSet<>();
+ Set<String> nonExistentLogins = new HashSet<>();
Set<String> unmappedIssueLabels = new HashSet<>();
+ Map<String, Pair<FieldSpec, String>> labelMappings = new HashMap<>();
+
+ for (IssueLabelMapping mapping: importOption.getIssueLabelMappings()) {
+ String oneDevFieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");
+ String oneDevFieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");
+ FieldSpec fieldSpec = getIssueSetting().getFieldSpec(oneDevFieldName);
+ if (fieldSpec == null)
+ throw new ExplicitException("No field spec found: " + oneDevFieldName);
+ labelMappings.put(mapping.getGitHubIssueLabel(), new Pair<>(fieldSpec, oneDevFieldValue));
+ }
+
Collection<Long> projectIds = new ArrayList<>();
try {
Map<String, Optional<User>> users = new HashMap<>();
+ boolean importIssues = false;
for (GitHubImport aImport: importOption.getImports()) {
logger.log("Cloning code of repository " + aImport.getGitHubRepo() + "...");
JsonNode repoNode = get(client, importSource, "/repos/" + aImport.getGitHubRepo(), logger);
@@ -123,6 +148,7 @@ public class GitHubImporter extends ProjectImporter<GitHubImportSource, GitHubIm
boolean isPrivate = repoNode.get("private").asBoolean();
if (!isPrivate && importOption.getPublicRole() != null)
project.setDefaultRole(importOption.getPublicRole());
+
URIBuilder builder = new URIBuilder(repoNode.get("clone_url").asText());
if (isPrivate)
builder.setUserInfo("git", importSource.getAccessToken());
@@ -130,17 +156,10 @@ public class GitHubImporter extends ProjectImporter<GitHubImportSource, GitHubIm
if (!dryRun) {
projectManager.clone(project, builder.build().toString());
projectIds.add(project.getId());
- } else {
- User user = SecurityUtils.getUser();
- UserAuthorization authorization = new UserAuthorization();
- authorization.setProject(project);
- authorization.setUser(user);
- authorization.setRole(OneDev.getInstance(RoleManager.class).getOwner());
- project.getUserAuthorizations().add(authorization);
- user.getAuthorizations().add(authorization);
}
if (aImport.isImportIssues()) {
+ importIssues = true;
Map<Long, Milestone> milestones = new HashMap<>();
logger.log("Importing milestones of repository " + aImport.getGitHubRepo() + "...");
String apiEndpoint = "/repos/" + aImport.getGitHubRepo() + "/milestones?state=all";
@@ -166,6 +185,68 @@ public class GitHubImporter extends ProjectImporter<GitHubImportSource, GitHubIm
AtomicInteger numOfImportedIssues = new AtomicInteger(0);
PageDataConsumer pageDataConsumer = new PageDataConsumer() {
+ /*
+ @Nullable
+ private String processAttachments(String issueUUID, Long issueNumber, @Nullable String markdown,
+ Set<String> tooLargeAttachments) {
+ if (markdown == null)
+ return markdown;
+
+ Map<String, String> attachments = new HashMap<>();
+
+ Node node = OneDev.getInstance(MarkdownManager.class).parse(markdown);
+ new NodeVisitor(new VisitHandler<?>[] {}) {
+
+ @Override
+ public void visit(Node node) {
+ super.visit(node);
+ if (node instanceof InlineLinkNode) {
+ InlineLinkNode link = (InlineLinkNode)node;
+ String url = link.getUrl().toString();
+ if (url.startsWith(repoUrl + "/files/")) {
+ attachments.put(url, link.getText().toString());
+ }
+ }
+ }
+
+ }.visit(node);
+
+ for (Map.Entry<String, String> entry: attachments.entrySet()) {
+ String attachmentUrl = entry.getKey();
+ String attachmentName = entry.getValue();
+
+ WebTarget target = client.target(attachmentUrl);
+ Invocation.Builder builder = target.request();
+ try (Response response = builder.get()) {
+ String errorMessage = JerseyUtils.checkStatus(response);
+ if (errorMessage != null) {
+ throw new ExplicitException(String.format(
+ "Error downloading attachment (url: %s, error message: %s)",
+ attachmentUrl, errorMessage));
+ }
+ try (InputStream is = response.readEntity(InputStream.class)) {
+ String oneDevAttachmentName = project.saveAttachment(issueUUID, attachmentName, is);
+ String oneDevAttachmentUrl = project.getAttachmentUrlPath(issueUUID, oneDevAttachmentName);
+ markdown = markdown.replace("(" + attachmentName + ")", "(" + oneDevAttachmentUrl + ")");
+ } catch (AttachmentTooLargeException e) {
+ tooLargeAttachments.add(aImport.getGitHubRepo() + "#" + issueNumber + ":" + attachmentName);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ return markdown;
+ }
+ */
+
+ private String joinAsMultilineHtml(List<String> values) {
+ List<String> escapedValues = new ArrayList<>();
+ for (String value: values)
+ escapedValues.add(HtmlEscape.escapeHtml5(value));
+ return StringUtils.join(escapedValues, "<br>");
+ }
+
@Override
public void consume(List<JsonNode> pageData) throws InterruptedException {
for (JsonNode issueNode: pageData) {
@@ -193,7 +274,7 @@ public class GitHubImporter extends ProjectImporter<GitHubImportSource, GitHubIm
issue.setSubmitter(user);
} else {
issue.setSubmitterName(login);
- loginsWithoutAccount.add(login);
+ nonExistentLogins.add(login);
}
issue.setSubmitDate(ISODateTimeFormat.dateTimeNoMillis()
@@ -207,8 +288,7 @@ public class GitHubImporter extends ProjectImporter<GitHubImportSource, GitHubIm
lastUpdate.setUserName(issue.getSubmitterName());
issue.setLastUpdate(lastUpdate);
- if (!dryRun)
- OneDev.getInstance(IssueManager.class).save(issue);
+ Map<String, String> extraIssueInfo = new LinkedHashMap<>();
for (JsonNode assigneeNode: issueNode.get("assignees")) {
IssueField assigneeField = new IssueField();
@@ -222,10 +302,9 @@ public class GitHubImporter extends ProjectImporter<GitHubImportSource, GitHubIm
assigneeField.setValue(user.getName());
} else {
assigneeField.setValue(login);
- loginsWithoutAccount.add(login);
+ nonExistentLogins.add(login);
}
- if (!dryRun)
- getIssueFieldManager().save(assigneeField);
+ issue.getFields().add(assigneeField);
}
String apiEndpoint = "/repos/" + aImport.getGitHubRepo()
@@ -244,39 +323,61 @@ public class GitHubImporter extends ProjectImporter<GitHubImportSource, GitHubIm
comment.setUser(user);
} else {
comment.setUserName(login);
- loginsWithoutAccount.add(login);
+ nonExistentLogins.add(login);
}
-
- if (!dryRun)
- OneDev.getInstance(IssueCommentManager.class).save(comment);
+
+ issue.getComments().add(comment);
}
+ issue.setCommentCount(issue.getComments().size());
+
apiEndpoint = "/repos/" + aImport.getGitHubRepo()
+ "/issues/" + issue.getNumber() + "/labels";
+ List<String> currentUnmappedLabels = new ArrayList<>();
for (JsonNode labelNode: list(client, importSource, apiEndpoint, logger)) {
String labelName = labelNode.get("name").asText();
- IssueField labelField = null;
- for (IssueLabelMapping mapping: importOption.getLabelMappings()) {
- if (mapping.getGitHubIssueLabel().equals(labelName)) {
- String fieldName = StringUtils.substringBefore(mapping.getOneDevIssueField(), "::");
- FieldSpec fieldSpec = getIssueSetting().getFieldSpecMap(null).get(fieldName);
- if (fieldSpec != null) {
- labelField = new IssueField();
- labelField.setIssue(issue);
- String fieldValue = StringUtils.substringAfter(mapping.getOneDevIssueField(), "::");
- labelField.setName(fieldName);
- labelField.setType(InputSpec.ENUMERATION);
- labelField.setValue(fieldValue);
- labelField.setOrdinal(fieldSpec.getOrdinal(fieldValue));
- if (!dryRun)
- getIssueFieldManager().save(labelField);
- }
- break;
- }
+ Pair<FieldSpec, String> mapped = labelMappings.get(labelName);
+ if (mapped != null) {
+ IssueField tagField = new IssueField();
+ tagField.setIssue(issue);
+ tagField.setName(mapped.getFirst().getName());
+ tagField.setType(InputSpec.ENUMERATION);
+ tagField.setValue(mapped.getSecond());
+ tagField.setOrdinal(mapped.getFirst().getOrdinal(mapped.getSecond()));
+ issue.getFields().add(tagField);
+ } else {
+ currentUnmappedLabels.add(HtmlEscape.escapeHtml5(labelName));
+ unmappedIssueLabels.add(HtmlEscape.escapeHtml5(labelName));
}
- if (labelField == null)
- unmappedIssueLabels.add(labelName);
}
+
+ if (!currentUnmappedLabels.isEmpty())
+ extraIssueInfo.put("Labels", joinAsMultilineHtml(currentUnmappedLabels));
+
+ if (!extraIssueInfo.isEmpty()) {
+ StringBuilder builder = new StringBuilder("|");
+ for (String key: extraIssueInfo.keySet())
+ builder.append(key).append("|");
+ builder.append("\n|");
+ extraIssueInfo.keySet().stream().forEach(it->builder.append("---|"));
+ builder.append("\n|");
+ for (String value: extraIssueInfo.values())
+ builder.append(value).append("|");
+
+ if (issue.getDescription() != null)
+ issue.setDescription(builder.toString() + "\n\n" + issue.getDescription());
+ else
+ issue.setDescription(builder.toString());
+ }
+
+ if (!dryRun) {
+ OneDev.getInstance(IssueManager.class).save(issue);
+ for (IssueField field: issue.getFields())
+ OneDev.getInstance(Dao.class).persist(field);
+ for (IssueComment comment: issue.getComments())
+ OneDev.getInstance(Dao.class).persist(comment);
+ }
+
if (numOfImportedIssues.incrementAndGet() % 25 == 0)
logger.log("Imported " + numOfImportedIssues.get() + " issues");
}
@@ -287,17 +388,29 @@ public class GitHubImporter extends ProjectImporter<GitHubImportSource, GitHubIm
list(client, importSource, apiEndpoint, pageDataConsumer, logger);
}
}
- List<String> result = new ArrayList<>();
- result.add("All repositories imported successfully");
- if (!loginsWithoutAccount.isEmpty()) {
- result.add("<b>NOTE:</b> GitHub logins without public email or public email can not be mapped to OneDev account: "
- + HtmlEscape.escapeHtml5(loginsWithoutAccount.toString()));
- }
+
+ StringBuilder feedback = new StringBuilder("Repositories imported successfully");
+
+ boolean hasNotes = !unmappedIssueLabels.isEmpty() || !nonExistentLogins.isEmpty() || importIssues;
+
+ if (hasNotes)
+ feedback.append("<br><br><b>NOTE:</b><ul>");
+
if (!unmappedIssueLabels.isEmpty()) {
- result.add("<b>NOTE:</b> Issue labels not mapped to custom field: "
+ feedback.append("<li> GitHub issue labels not mapped to OneDev custom field (mentioned as extra info in issue description): "
+ HtmlEscape.escapeHtml5(unmappedIssueLabels.toString()));
}
- return StringUtils.join(result, "<br>");
+ if (!nonExistentLogins.isEmpty()) {
+ feedback.append("<li> GitHub logins without public email or public email can not be mapped to OneDev account: "
+ + HtmlEscape.escapeHtml5(nonExistentLogins.toString()));
+ }
+ if (importIssues)
+ feedback.append("<li> Attachments in issues and comments are not imported due to GitHub limitation");
+
+ if (hasNotes)
+ feedback.append("</ul>");
+
+ return feedback.toString();
} catch (Exception e) {
for (Long projectId: projectIds)
storageManager.deleteProjectDir(projectId);
@@ -311,10 +424,6 @@ public class GitHubImporter extends ProjectImporter<GitHubImportSource, GitHubIm
return OneDev.getInstance(SettingManager.class).getIssueSetting();
}
- private IssueFieldManager getIssueFieldManager() {
- return OneDev.getInstance(IssueFieldManager.class);
- }
-
private List<JsonNode> list(Client client, GitHubImportSource importSource,
String apiPath, SimpleLogger logger) {
List<JsonNode> result = new ArrayList<>();
@@ -384,8 +493,7 @@ public class GitHubImporter extends ProjectImporter<GitHubImportSource, GitHubIm
status, errorMessage));
}
} else {
- throw new RuntimeException("Http request failed with status " + status
- + ", check server log for detaiils");
+ throw new ExplicitException(String.format("Http request failed (status: %s)", status));
}
}
return response.readEntity(JsonNode.class);
@@ -395,6 +503,7 @@ public class GitHubImporter extends ProjectImporter<GitHubImportSource, GitHubIm
private Client newClient(GitHubImportSource importSource) {
Client client = ClientBuilder.newClient();
+ client.property(ClientProperties.FOLLOW_REDIRECTS, true);
client.register(HttpAuthenticationFeature.basic("git", importSource.getAccessToken()));
return client;
}
diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportOption.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportOption.java
index f3ce34d6ac..b03e6e16a9 100644
--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportOption.java
+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportOption.java
@@ -60,7 +60,8 @@ public class YouTrackImportOption implements Serializable, Validatable {
}
@Editable(order=300, description="Specify how to map YouTrack issue tags to OneDev issue custom "
- + "fields. Unmapped tags will be reflected in issue description")
+ + "fields. Only multi-valued enum field can be used here. Unmapped tags will be "
+ + "reflected in issue description")
public List<IssueTagMapping> getIssueTagMappings() {
return issueTagMappings;
}
diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportSource.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportSource.java
index ef06b754fd..e7d052383e 100644
--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportSource.java
+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImportSource.java
@@ -15,6 +15,7 @@ import org.hibernate.validator.constraints.NotEmpty;
import com.fasterxml.jackson.databind.JsonNode;
import io.onedev.commons.utils.StringUtils;
+import io.onedev.server.util.JerseyUtils;
import io.onedev.server.util.validation.Validatable;
import io.onedev.server.util.validation.annotation.ClassValidating;
import io.onedev.server.web.editable.annotation.Editable;
@@ -77,8 +78,7 @@ public class YouTrackImportSource implements Serializable, Validatable {
}
@Editable(order=400, description="If checked, import options will be pre-populated based on all accessible "
- + "projects and its settings. In case there are too many projects and settings, you may want to "
- + "uncheck this and provide import option manually")
+ + "projects and its settings")
public boolean isPrepopulateImportOptions() {
return prepopulateImportOptions;
}
@@ -99,7 +99,7 @@ public class YouTrackImportSource implements Serializable, Validatable {
WebTarget target = client.target(getApiEndpoint("/users/me?fields=guest"));
Invocation.Builder builder = target.request();
try (Response response = builder.get()) {
- String errorMessage = YouTrackImporter.checkStatus(response);
+ String errorMessage = JerseyUtils.checkStatus(response);
if (errorMessage != null) {
context.disableDefaultConstraintViolation();
context.buildConstraintViolationWithTemplate(errorMessage)
diff --git a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImporter.java b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImporter.java
index 8c54deafb1..9f3237b82d 100644
--- a/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImporter.java
+++ b/server-plugin/server-plugin-import-youtrack/src/main/java/io/onedev/server/plugin/imports/youtrack/YouTrackImporter.java
@@ -58,6 +58,7 @@ import io.onedev.server.persistence.dao.Dao;
import io.onedev.server.security.SecurityUtils;
import io.onedev.server.storage.StorageManager;
import io.onedev.server.util.DateUtils;
+import io.onedev.server.util.JerseyUtils;
import io.onedev.server.util.Pair;
import io.onedev.server.util.SimpleLogger;
import io.onedev.server.web.page.project.imports.ProjectImporter;
@@ -182,7 +183,7 @@ public class YouTrackImporter extends ProjectImporter<YouTrackImportSource, YouT
projectDescriptions.put(projectName, projectNode.get("description").asText(null));
}
- Set<String> unmappedAccounts = new LinkedHashSet<>();
+ Set<String> nonExistentLogins = new LinkedHashSet<>();
Set<String> unmappedIssueTags = new LinkedHashSet<>();
Set<String> unmappedIssueFields = new LinkedHashSet<>();
Set<String> unmappedIssueStates = new LinkedHashSet<>();
@@ -241,32 +242,28 @@ public class YouTrackImporter extends ProjectImporter<YouTrackImportSource, YouT
if (!attachmentUrl.startsWith("/api"))
throw new ExplicitException("Unexpected attachment url: " + attachmentUrl);
attachmentUrl = attachmentUrl.substring("/api".length());
-
- WebTarget target = client.target(importSource.getApiEndpoint(attachmentUrl));
+
+ String endpoint = importSource.getApiEndpoint(attachmentUrl);
+ WebTarget target = client.target(endpoint);
Invocation.Builder builder = target.request();
try (Response response = builder.get()) {
- int status = response.getStatus();
- if (status != 200) {
- String errorMessage = response.readEntity(String.class);
- if (StringUtils.isNotBlank(errorMessage)) {
- throw new ExplicitException("Error downloading attachment: " + errorMessage);
- } else {
- throw new RuntimeException("Attachment download failed with status " + status
- + ", check server log for detaiils");
- }
- } else {
- try (InputStream is = response.readEntity(InputStream.class)) {
- String oneDevAttachmentName = project.saveAttachment(issueUUID, attachmentName, is);
- String oneDevAttachmentUrl = project.getAttachmentUrlPath(issueUUID, oneDevAttachmentName);
- if (markdown.contains("(" + attachmentName + ")")) {
- markdown = markdown.replace("(" + attachmentName + ")", "(" + oneDevAttachmentUrl + ")");
- } else {
- unreferencedAttachments.put(attachmentName, oneDevAttachmentUrl);
- }
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
+ String errorMessage = JerseyUtils.checkStatus(response);
+ if (errorMessage != null) {
+ throw new ExplicitException(String.format(
+ "Error downloading attachment (url: %s, error message: %s)",
+ endpoint, errorMessage));
}
+ try (InputStream is = response.readEntity(InputStream.class)) {
+ String oneDevAttachmentName = project.saveAttachment(issueUUID, attachmentName, is);
+ String oneDevAttachmentUrl = project.getAttachmentUrlPath(issueUUID, oneDevAttachmentName);
+ if (markdown.contains("(" + attachmentName + ")")) {
+ markdown = markdown.replace("(" + attachmentName + ")", "(" + oneDevAttachmentUrl + ")");
+ } else {
+ unreferencedAttachments.put(attachmentName, oneDevAttachmentUrl);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
}
}
}
@@ -338,11 +335,11 @@ public class YouTrackImporter extends ProjectImporter<YouTrackImportSource, YouT
issue.setSubmitter(user);
} else {
issue.setSubmitterName(fullName);
- unmappedAccounts.add(login);
+ nonExistentLogins.add(login);
}
} else {
issue.setSubmitterName(fullName);
- unmappedAccounts.add(login);
+ nonExistentLogins.add(login);
}
} else {
issue.setSubmitter(SecurityUtils.getUser());
@@ -491,11 +488,11 @@ public class YouTrackImporter extends ProjectImporter<YouTrackImportSource, YouT
fieldValue = user.getName();
} else {
fieldValue = fullName;
- unmappedAccounts.add(login);
+ nonExistentLogins.add(login);
}
} else {
fieldValue = fullName;
- unmappedAccounts.add(login);
+ nonExistentLogins.add(login);
}
IssueField issueField = new IssueField();
@@ -531,11 +528,11 @@ public class YouTrackImporter extends ProjectImporter<YouTrackImportSource, YouT
fieldValue = user.getName();
} else {
fieldValue = fullName;
- unmappedAccounts.add(login);
+ nonExistentLogins.add(login);
}
} else {
fieldValue = fullName;
- unmappedAccounts.add(login);
+ nonExistentLogins.add(login);
}
IssueField issueField = new IssueField();
@@ -763,11 +760,11 @@ public class YouTrackImporter extends ProjectImporter<YouTrackImportSource, YouT
comment.setUser(user);
} else {
comment.setUserName(fullName);
- unmappedAccounts.add(login);
+ nonExistentLogins.add(login);
}
} else {
comment.setUserName(fullName);
- unmappedAccounts.add(login);
+ nonExistentLogins.add(login);
}
} else {
issue.setSubmitter(SecurityUtils.getUser());
@@ -817,7 +814,7 @@ public class YouTrackImporter extends ProjectImporter<YouTrackImportSource, YouT
|| !unmappedIssueFields.isEmpty()
|| !mismatchedIssueFields.isEmpty()
|| !unmappedIssueTags.isEmpty()
- || !unmappedAccounts.isEmpty()
+ || !nonExistentLogins.isEmpty()
|| !tooLargeAttachments.isEmpty();
if (hasNotes)
@@ -846,9 +843,9 @@ public class YouTrackImporter extends ProjectImporter<YouTrackImportSource, YouT
feedback.append("<li> YouTrack issue tags not mapped to OneDev custom field (mentioned as extra info in issue description): "
+ HtmlEscape.escapeHtml5(unmappedIssueTags.toString()));
}
- if (!unmappedAccounts.isEmpty()) {
+ if (!nonExistentLogins.isEmpty()) {
feedback.append("<li> YouTrack logins without email or email can not be mapped to OneDev account: "
- + HtmlEscape.escapeHtml5(unmappedAccounts.toString()));
+ + HtmlEscape.escapeHtml5(nonExistentLogins.toString()));
}
if (!tooLargeAttachments.isEmpty()) {
feedback.append("<li> Too large attachments: "
@@ -918,7 +915,7 @@ public class YouTrackImporter extends ProjectImporter<YouTrackImportSource, YouT
WebTarget target = client.target(apiEndpoint);
Invocation.Builder builder = target.request();
try (Response response = builder.get()) {
- String errorMessage = checkStatus(response);
+ String errorMessage = JerseyUtils.checkStatus(response);
if (errorMessage != null)
throw new ExplicitException(errorMessage);
else
@@ -926,22 +923,6 @@ public class YouTrackImporter extends ProjectImporter<YouTrackImportSource, YouT
}
}
- @Nullable
- public static String checkStatus(Response response) {
- int status = response.getStatus();
- if (status != 200) {
- String errorMessage = response.readEntity(String.class);
- if (StringUtils.isNotBlank(errorMessage)) {
- return String.format("Http request failed (status code: %d, error message: %s)",
- status, errorMessage);
- } else {
- return String.format("Http request failed (status code: %d)", status);
- }
- } else {
- return null;
- }
- }
-
private Client newClient(YouTrackImportSource importSource) {
Client client = ClientBuilder.newClient();
client.register(HttpAuthenticationFeature.basic(importSource.getUserName(), importSource.getPassword()));