Skip to content

The Unified and Autonomous Development Platform

Refactor commit info collecting to make it more performant and accuracy

Robin Shenafcea0748294
diff --git a/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultCodeCommentManager.java b/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultCodeCommentManager.java
index 919ffcd664..c816a05967 100644
--- a/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultCodeCommentManager.java
+++ b/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultCodeCommentManager.java
@@ -35,6 +35,7 @@ import org.hibernate.query.Query;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
import io.onedev.commons.launcher.loader.Listen;
import io.onedev.commons.launcher.loader.ListenerRegistry;
@@ -149,11 +150,13 @@ public class DefaultCodeCommentManager extends AbstractEntityManager<CodeComment
Map<CodeComment, PlanarRange> comments = new HashMap<>();
Map<String, Map<String, List<CodeComment>>> possibleComments = new HashMap<>();
- for (String possibleHistoryPath: commitInfoManager.getHistoryPaths(project, path)) {
+ Collection<String> possiblePaths = Sets.newHashSet(path);
+ possiblePaths.addAll(commitInfoManager.getHistoryPaths(project, path));
+ for (String possiblePath: possiblePaths) {
EntityCriteria<CodeComment> criteria = EntityCriteria.of(CodeComment.class);
- criteria.add(Restrictions.eq("markPos.path", possibleHistoryPath));
+ criteria.add(Restrictions.eq("markPos.path", possiblePath));
for (CodeComment comment: query(criteria)) {
- if (comment.getMarkPos().getCommit().equals(commitId.name()) && possibleHistoryPath.equals(path)) {
+ if (comment.getMarkPos().getCommit().equals(commitId.name()) && possiblePath.equals(path)) {
comments.put(comment, comment.getMarkPos().getRange());
} else {
Map<String, List<CodeComment>> commentsOnCommit =
@@ -162,10 +165,10 @@ public class DefaultCodeCommentManager extends AbstractEntityManager<CodeComment
commentsOnCommit = new HashMap<>();
possibleComments.put(comment.getMarkPos().getCommit(), commentsOnCommit);
}
- List<CodeComment> commentsOnPath = commentsOnCommit.get(possibleHistoryPath);
+ List<CodeComment> commentsOnPath = commentsOnCommit.get(possiblePath);
if (commentsOnPath == null) {
commentsOnPath = new ArrayList<>();
- commentsOnCommit.put(possibleHistoryPath, commentsOnPath);
+ commentsOnCommit.put(possiblePath, commentsOnPath);
}
commentsOnPath.add(comment);
}
diff --git a/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultPullRequestManager.java b/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultPullRequestManager.java
index 9dd9fe2043..3f88debd5f 100644
--- a/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultPullRequestManager.java
+++ b/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultPullRequestManager.java
@@ -84,7 +84,6 @@ import io.onedev.server.event.pullrequest.PullRequestEvent;
import io.onedev.server.event.pullrequest.PullRequestMergePreviewCalculated;
import io.onedev.server.event.pullrequest.PullRequestOpened;
import io.onedev.server.git.GitUtils;
-import io.onedev.server.git.command.FileChange;
import io.onedev.server.infomanager.CommitInfoManager;
import io.onedev.server.model.Build;
import io.onedev.server.model.Group;
@@ -140,7 +139,7 @@ public class DefaultPullRequestManager extends AbstractEntityManager<PullRequest
private static final Logger logger = LoggerFactory.getLogger(DefaultPullRequestManager.class);
- private static final int MAX_CONTRIBUTION_FILES = 100;
+ private static final int MAX_COMMIT_FILES = 100;
private static final int UI_PREVIEW_PRIORITY = 10;
@@ -842,7 +841,7 @@ public class DefaultPullRequestManager extends AbstractEntityManager<PullRequest
candidateReviewers.add(user);
}
- sortByContributions(candidateReviewers, update);
+ sortByCommitCount(candidateReviewers, update);
for (User user: candidateReviewers) {
reviewers.add(user);
@@ -858,7 +857,7 @@ public class DefaultPullRequestManager extends AbstractEntityManager<PullRequest
if (review == null)
candidateReviewers.add(user);
}
- sortByContributions(candidateReviewers, update);
+ sortByCommitCount(candidateReviewers, update);
for (User user: candidateReviewers) {
reviewers.add(user);
PullRequestReview review = new PullRequestReview();
@@ -894,34 +893,28 @@ public class DefaultPullRequestManager extends AbstractEntityManager<PullRequest
}
}
- private void sortByContributions(List<User> users, PullRequestUpdate update) {
+ private void sortByCommitCount(List<User> users, PullRequestUpdate update) {
if (users.size() <= 1)
return;
CommitInfoManager commitInfoManager = OneDev.getInstance(CommitInfoManager.class);
- Map<User, Long> contributions = new HashMap<>();
+ Map<User, Long> commitCounts = new HashMap<>();
for (User user: users)
- contributions.put(user, 0L);
+ commitCounts.put(user, 0L);
int count = 0;
- for (FileChange change: update.getFileChanges()) {
- String path = change.getPath();
- int edits = change.getAdditions() + change.getDeletions();
- if (edits < 0)
- edits = 100;
- else if (edits == 0)
- edits = 1;
- int addedContributions = addContributions(contributions, commitInfoManager, path, edits, update);
- while (addedContributions == 0) {
+ for (String path: update.getChangedFiles()) {
+ int addedCommitCount = addCommitCounts(commitCounts, commitInfoManager, path, update);
+ while (addedCommitCount == 0) {
if (path.contains("/")) {
path = StringUtils.substringBeforeLast(path, "/");
- addedContributions = addContributions(contributions, commitInfoManager, path, edits, update);
+ addedCommitCount = addCommitCounts(commitCounts, commitInfoManager, path, update);
} else {
- addContributions(contributions, commitInfoManager, "", edits, update);
+ addCommitCounts(commitCounts, commitInfoManager, "", update);
break;
}
}
- if (++count >= MAX_CONTRIBUTION_FILES)
+ if (++count >= MAX_COMMIT_FILES)
break;
}
@@ -929,7 +922,7 @@ public class DefaultPullRequestManager extends AbstractEntityManager<PullRequest
@Override
public int compare(User o1, User o2) {
- if (contributions.get(o1) < contributions.get(o2))
+ if (commitCounts.get(o1) < commitCounts.get(o2))
return 1;
else
return -1;
@@ -938,17 +931,16 @@ public class DefaultPullRequestManager extends AbstractEntityManager<PullRequest
});
}
- private int addContributions(Map<User, Long> contributions, CommitInfoManager commitInfoManager,
- String path, int edits, PullRequestUpdate update) {
- int addedContributions = 0;
- for (Map.Entry<User, Long> entry: contributions.entrySet()) {
+ private int addCommitCounts(Map<User, Long> commitCounts, CommitInfoManager commitInfoManager,
+ String path, PullRequestUpdate update) {
+ int addedCommitCount = 0;
+ for (Map.Entry<User, Long> entry: commitCounts.entrySet()) {
User user = entry.getKey();
- int pathEdits = commitInfoManager.getEdits(
- update.getRequest().getTargetProject(), user, path);
- entry.setValue(entry.getValue() + edits*pathEdits);
- addedContributions += pathEdits;
+ int commitCount = commitInfoManager.getCommitCount(update.getRequest().getTargetProject(), user, path);
+ entry.setValue(entry.getValue() + commitCount);
+ addedCommitCount += commitCount;
}
- return addedContributions;
+ return addedCommitCount;
}
private Predicate[] getPredicates(@Nullable Project targetProject,
diff --git a/server-core/src/main/java/io/onedev/server/git/GitPostReceiveCallback.java b/server-core/src/main/java/io/onedev/server/git/GitPostReceiveCallback.java
index 07cf4d53cf..bebf83da92 100644
--- a/server-core/src/main/java/io/onedev/server/git/GitPostReceiveCallback.java
+++ b/server-core/src/main/java/io/onedev/server/git/GitPostReceiveCallback.java
@@ -114,7 +114,7 @@ public class GitPostReceiveCallback extends HttpServlet {
RefUpdate refUpdate = GitUtils.getRefUpdate(project.getRepository(), "HEAD");
GitUtils.linkRef(refUpdate, refName);
}
-
+
listenerRegistry.post(new RefUpdated(project, refName, oldObjectId, newObjectId));
field = field.substring(40);
diff --git a/server-core/src/main/java/io/onedev/server/git/command/FileChange.java b/server-core/src/main/java/io/onedev/server/git/command/FileChange.java
index 63a6151488..a39f1a61f4 100644
--- a/server-core/src/main/java/io/onedev/server/git/command/FileChange.java
+++ b/server-core/src/main/java/io/onedev/server/git/command/FileChange.java
@@ -1,31 +1,38 @@
package io.onedev.server.git.command;
import java.io.Serializable;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
+import javax.annotation.Nullable;
@SuppressWarnings("serial")
public class FileChange implements Serializable {
private final String oldPath;
- private final String path;
+ private final String newPath;
private final int additions;
private final int deletions;
- public FileChange(String oldPath, String path, int additions, int deletions) {
+ public FileChange(@Nullable String oldPath, @Nullable String newPath, int additions, int deletions) {
this.oldPath = oldPath;
- this.path = path;
+ this.newPath = newPath;
this.additions = additions;
this.deletions = deletions;
}
+ @Nullable
public String getOldPath() {
return oldPath;
}
- public String getPath() {
- return path;
+ @Nullable
+ public String getNewPath() {
+ return newPath;
}
public int getAdditions() {
@@ -36,12 +43,13 @@ public class FileChange implements Serializable {
return deletions;
}
- @Override
- public String toString() {
- if (oldPath != null)
- return oldPath + "=>" + path;
- else
- return path;
+ public Collection<String> getPaths() {
+ Set<String> paths = new HashSet<>();
+ if (getOldPath() != null)
+ paths.add(getOldPath());
+ if (getNewPath() != null)
+ paths.add(getNewPath());
+ return paths;
}
}
\ No newline at end of file
diff --git a/server-core/src/main/java/io/onedev/server/git/command/GitCommand.java b/server-core/src/main/java/io/onedev/server/git/command/GitCommand.java
index ff47c3e5f8..4114b33d38 100644
--- a/server-core/src/main/java/io/onedev/server/git/command/GitCommand.java
+++ b/server-core/src/main/java/io/onedev/server/git/command/GitCommand.java
@@ -2,10 +2,12 @@ package io.onedev.server.git.command;
import java.io.File;
import java.util.Map;
+import java.util.StringTokenizer;
import javax.annotation.Nullable;
import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.eclipse.jgit.util.QuotedString;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -97,6 +99,40 @@ public abstract class GitCommand<V> {
return cmd;
}
+ public static FileChange parseNumStats(String line) {
+ FileChange change;
+ StringTokenizer tokenizer = new StringTokenizer(line, "\t");
+ String additionsToken = tokenizer.nextToken();
+ int additions = additionsToken.equals("-")?-1:Integer.parseInt(additionsToken);
+ String deletionsToken = tokenizer.nextToken();
+ int deletions = deletionsToken.equals("-")?-1:Integer.parseInt(deletionsToken);
+
+ String path = tokenizer.nextToken();
+ int renameSignIndex = path.indexOf(" => ");
+ if (renameSignIndex != -1) {
+ int leftBraceIndex = path.indexOf("{");
+ int rightBraceIndex = path.indexOf("}");
+ if (leftBraceIndex != -1 && rightBraceIndex != -1 && leftBraceIndex<renameSignIndex
+ && rightBraceIndex>renameSignIndex) {
+ String leftCommon = path.substring(0, leftBraceIndex);
+ String rightCommon = path.substring(rightBraceIndex+1);
+ String oldPath = leftCommon + path.substring(leftBraceIndex+1, renameSignIndex)
+ + rightCommon;
+ String newPath = leftCommon + path.substring(renameSignIndex+4, rightBraceIndex)
+ + rightCommon;
+ change = new FileChange(oldPath, newPath, additions, deletions);
+ } else {
+ String oldPath = QuotedString.GIT_PATH.dequote(path.substring(0, renameSignIndex));
+ String newPath = QuotedString.GIT_PATH.dequote(path.substring(renameSignIndex+4));
+ change = new FileChange(oldPath, newPath, additions, deletions);
+ }
+ } else {
+ path = QuotedString.GIT_PATH.dequote(path);
+ change = new FileChange(null, path, additions, deletions);
+ }
+ return change;
+ }
+
protected String getGitExe() {
return AppLoader.getInstance(GitConfig.class).getExecutable();
}
diff --git a/server-core/src/main/java/io/onedev/server/git/command/GitCommit.java b/server-core/src/main/java/io/onedev/server/git/command/GitCommit.java
index e91e9876e0..0ba4adbb18 100644
--- a/server-core/src/main/java/io/onedev/server/git/command/GitCommit.java
+++ b/server-core/src/main/java/io/onedev/server/git/command/GitCommit.java
@@ -11,8 +11,6 @@ import javax.annotation.Nullable;
import org.apache.commons.lang3.StringUtils;
import org.eclipse.jgit.lib.PersonIdent;
-import com.google.common.base.Preconditions;
-
import io.onedev.server.git.GitUtils;
public class GitCommit implements Serializable {
@@ -21,6 +19,10 @@ public class GitCommit implements Serializable {
private final String hash;
+ private final String subject;
+
+ private final String body;
+
private final Date commitDate;
private final PersonIdent committer;
@@ -31,30 +33,44 @@ public class GitCommit implements Serializable {
private final List<FileChange> fileChanges;
- public GitCommit(String hash, @Nullable Date commitDate, @Nullable PersonIdent committer,
- @Nullable PersonIdent author, List<String> parentHashes, @Nullable List<FileChange> fileChanges) {
- this.hash = Preconditions.checkNotNull(hash);
- this.commitDate = commitDate;
+ public GitCommit(String hash, @Nullable List<String> parentHashes, @Nullable PersonIdent committer,
+ @Nullable PersonIdent author, @Nullable Date commitDate, @Nullable String subject,
+ @Nullable String body, @Nullable List<FileChange> fileChanges) {
+ this.hash = hash;
+ this.parentHashes = parentHashes;
this.committer = committer;
this.author = author;
- this.parentHashes = Preconditions.checkNotNull(parentHashes);
+ this.commitDate = commitDate;
+ this.subject = subject;
+ this.body = body;
this.fileChanges = fileChanges;
}
-
+
+ @Nullable
+ public String getSubject() {
+ return subject;
+ }
+
+ @Nullable
+ public String getBody() {
+ return body;
+ }
+
+ @Nullable
public List<String> getParentHashes() {
return parentHashes;
}
- @Nullable
+ @Nullable
public List<FileChange> getFileChanges() {
return fileChanges;
}
- public Collection<String> getChangedFiles() {
+ public Collection<String> getChangedFiles() {
Collection<String> changedFiles = new HashSet<>();
for (FileChange change: getFileChanges()) {
- if (change.getPath() != null)
- changedFiles.add(change.getPath());
+ if (change.getNewPath() != null)
+ changedFiles.add(change.getNewPath());
if (change.getOldPath() != null)
changedFiles.add(change.getOldPath());
}
@@ -65,7 +81,7 @@ public class GitCommit implements Serializable {
return hash;
}
- @Nullable
+ @Nullable
public PersonIdent getCommitter() {
return committer;
}
@@ -102,6 +118,10 @@ public class GitCommit implements Serializable {
public String hash;
+ public String subject;
+
+ public String body;
+
public String authorName;
public String authorEmail;
@@ -114,6 +134,8 @@ public class GitCommit implements Serializable {
public Date committerDate;
+ public Date commitDate;
+
public List<String> parentHashes;
public List<FileChange> fileChanges;
@@ -130,8 +152,15 @@ public class GitCommit implements Serializable {
author = GitUtils.newPersonIdent(authorName, authorEmail, authorDate);
else
author = null;
+
+ if (subject != null)
+ subject = subject.trim();
+
+ if (body != null)
+ body = body.trim();
- return new GitCommit(hash, committerDate, committer, author, parentHashes, fileChanges);
+ return new GitCommit(hash, parentHashes, committer, author, commitDate,
+ subject, body, fileChanges);
}
}
diff --git a/server-core/src/main/java/io/onedev/server/git/command/ListFileChangesCommand.java b/server-core/src/main/java/io/onedev/server/git/command/ListFileChangesCommand.java
deleted file mode 100644
index 7f66c723a2..0000000000
--- a/server-core/src/main/java/io/onedev/server/git/command/ListFileChangesCommand.java
+++ /dev/null
@@ -1,99 +0,0 @@
-package io.onedev.server.git.command;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.StringTokenizer;
-
-import org.eclipse.jgit.util.QuotedString;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-
-import io.onedev.commons.utils.command.Commandline;
-import io.onedev.commons.utils.command.LineConsumer;
-
-public class ListFileChangesCommand extends GitCommand<Collection<FileChange>> {
-
- private static final Logger logger = LoggerFactory.getLogger(ListFileChangesCommand.class);
-
- private String fromRev;
-
- private String toRev;
-
- private String path;
-
- public ListFileChangesCommand(File gitDir) {
- super(gitDir);
- }
-
- public ListFileChangesCommand fromRev(String fromRev) {
- this.fromRev = fromRev;
- return this;
- }
-
- public ListFileChangesCommand toRev(String toRev) {
- this.toRev = toRev;
- return this;
- }
-
- public ListFileChangesCommand path(String path) {
- this.path = path;
- return this;
- }
-
- @Override
- public Collection<FileChange> call() {
- Preconditions.checkNotNull(toRev, "toRev has to be specified.");
- Preconditions.checkNotNull(fromRev, "fromRev has to be specified.");
-
- final Collection<FileChange> fileChanges = new ArrayList<>();
-
- Commandline cmd = cmd();
-
- cmd.addArgs("diff", "--numstat", "--no-renames", fromRev + ".." + toRev);
-
- if (path != null)
- cmd.addArgs("--", path);
-
- cmd.execute(new LineConsumer() {
-
- @Override
- public void consume(String line) {
- if (line.contains("\t")) {
- StringTokenizer tokenizer = new StringTokenizer(line, "\t");
-
- int additions;
- String field = tokenizer.nextToken();
- if (field.equals("-"))
- additions = -1;
- else
- additions = Integer.parseInt(field);
-
- int deletions;
- field = tokenizer.nextToken();
- if (field.equals("-"))
- deletions = -1;
- else
- deletions = Integer.parseInt(field);
-
- String file = QuotedString.GIT_PATH.dequote(tokenizer.nextToken());
-
- fileChanges.add(new FileChange(null, file, additions, deletions));
- }
- }
-
- }, new LineConsumer() {
-
- @Override
- public void consume(String line) {
- logger.error(line);
- }
-
- }).checkReturnCode();
-
- return fileChanges;
- }
-
-}
diff --git a/server-core/src/main/java/io/onedev/server/git/command/ListNumStatsCommand.java b/server-core/src/main/java/io/onedev/server/git/command/ListNumStatsCommand.java
new file mode 100644
index 0000000000..23ae91fa7d
--- /dev/null
+++ b/server-core/src/main/java/io/onedev/server/git/command/ListNumStatsCommand.java
@@ -0,0 +1,68 @@
+package io.onedev.server.git.command;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+
+import io.onedev.commons.utils.command.Commandline;
+import io.onedev.commons.utils.command.LineConsumer;
+
+public class ListNumStatsCommand extends GitCommand<List<FileChange>> {
+
+ private static final Logger logger = LoggerFactory.getLogger(ListNumStatsCommand.class);
+
+ private String fromRev;
+
+ private String toRev;
+
+ public ListNumStatsCommand(File gitDir) {
+ super(gitDir);
+ }
+
+ public ListNumStatsCommand fromRev(String fromRev) {
+ this.fromRev = fromRev;
+ return this;
+ }
+
+ public ListNumStatsCommand toRev(String toRev) {
+ this.toRev = toRev;
+ return this;
+ }
+
+ @Override
+ public List<FileChange> call() {
+ Preconditions.checkNotNull(toRev, "toRev has to be specified.");
+ Preconditions.checkNotNull(fromRev, "fromRev has to be specified.");
+
+ List<FileChange> fileChanges = new ArrayList<>();
+
+ Commandline cmd = cmd();
+
+ cmd.addArgs("-c", "diff.renameLimit=1000", "diff", "--numstat",
+ "--find-renames", fromRev + ".." + toRev);
+
+ cmd.execute(new LineConsumer() {
+
+ @Override
+ public void consume(String line) {
+ fileChanges.add(parseNumStats(line));
+ }
+
+ }, new LineConsumer() {
+
+ @Override
+ public void consume(String line) {
+ logger.error(line);
+ }
+
+ }).checkReturnCode();
+
+ return fileChanges;
+ }
+
+}
\ No newline at end of file
diff --git a/server-core/src/main/java/io/onedev/server/git/command/LogCommand.java b/server-core/src/main/java/io/onedev/server/git/command/LogCommand.java
index 77bd94b87e..5b720b0061 100644
--- a/server-core/src/main/java/io/onedev/server/git/command/LogCommand.java
+++ b/server-core/src/main/java/io/onedev/server/git/command/LogCommand.java
@@ -5,6 +5,7 @@ import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
import java.util.StringTokenizer;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import org.eclipse.jgit.util.QuotedString;
@@ -22,9 +23,13 @@ public abstract class LogCommand extends GitCommand<Void> {
private static final Logger logger = LoggerFactory.getLogger(LogCommand.class);
- public enum Field {PARENTS, AUTHOR, COMMITTER, FILE_CHANGES};
+ public enum Field {PARENTS, AUTHOR, COMMITTER, COMMIT_DATE, SUBJECT, BODY, FILE_CHANGES, LINE_CHANGES};
- private EnumSet<Field> fields = EnumSet.allOf(Field.class);
+ private static final String BODY_END = "$<#BodyEnd#>$";
+
+ private boolean firstParent;
+
+ private EnumSet<Field> fields = EnumSet.noneOf(Field.class);
private List<String> revisions = new ArrayList<>();
@@ -41,15 +46,16 @@ public abstract class LogCommand extends GitCommand<Void> {
return this;
}
- public EnumSet<Field> fields() {
- return fields;
- }
-
public LogCommand fields(EnumSet<Field> fields) {
this.fields = fields;
return this;
}
-
+
+ public LogCommand firstParent(boolean firstParent) {
+ this.firstParent = firstParent;
+ return this;
+ }
+
@Override
public Void call() {
Preconditions.checkArgument(!revisions.isEmpty(), "Log revisions have to be specified");
@@ -73,31 +79,65 @@ public abstract class LogCommand extends GitCommand<Void> {
+ "committerEmail:%cE %n"
+ "committerDate:%cd %n";
}
-
- if (fields.contains(Field.FILE_CHANGES))
+ if (fields.contains(Field.COMMIT_DATE))
+ format += "commitDate:%cd %n";
+ if (fields.contains(Field.SUBJECT))
+ format += "subject:%s %n";
+ if (fields.contains(Field.BODY))
+ format += "body:%b" + BODY_END + "%n";
+
+ if (fields.contains(Field.LINE_CHANGES))
cmd.addArgs("-c", "diff.renameLimit=1000", "log", "--numstat", "--find-renames");
+ else if (fields.contains(Field.FILE_CHANGES))
+ cmd.addArgs("-c", "diff.renameLimit=1000", "log", "--name-status", "--find-renames");
else
cmd.addArgs("log");
cmd.addArgs("--format=" + format, "--date=raw");
-
+ if (firstParent) {
+ cmd.addArgs("--first-parent");
+ if (fields.contains(Field.LINE_CHANGES) || fields.contains(Field.FILE_CHANGES))
+ cmd.addArgs("-m");
+ }
+
for (String revision: revisions)
cmd.addArgs(revision);
AtomicReference<GitCommit.Builder> commitBuilderRef = new AtomicReference<>(null);
+ AtomicBoolean inBodyRef = new AtomicBoolean(false);
cmd.execute(new LineConsumer() {
@Override
public void consume(String line) {
- if (line.startsWith("hash:")) {
+ if (inBodyRef.get()) {
+ line = line.trim();
+ if (line.endsWith(BODY_END)) {
+ commitBuilderRef.get().body += "\n" + line.substring(0, line.length()-BODY_END.length());
+ inBodyRef.set(false);
+ } else {
+ commitBuilderRef.get().body += "\n" + line;
+ }
+ } else if (line.startsWith("body:")) {
+ line = line.substring("body:".length()).trim();
+ if (line.endsWith(BODY_END)) {
+ commitBuilderRef.get().body = line.substring(0, line.length()-BODY_END.length());
+ } else {
+ commitBuilderRef.get().body = line;
+ inBodyRef.set(true);
+ }
+ } else if (line.startsWith("hash:")) {
if (commitBuilderRef.get() != null)
LogCommand.this.consume(commitBuilderRef.get().build());
commitBuilderRef.set(new GitCommit.Builder());
if (fields.contains(Field.PARENTS))
commitBuilderRef.get().parentHashes = new ArrayList<>();
- if (fields.contains(Field.FILE_CHANGES))
+ if (fields.contains(Field.FILE_CHANGES) || fields.contains(Field.LINE_CHANGES))
commitBuilderRef.get().fileChanges = new ArrayList<>();
commitBuilderRef.get().hash = line.substring("hash:".length()).trim();
+ } else if (line.startsWith("parents:")) {
+ Splitter splitter = Splitter.on(" ").omitEmptyStrings().trimResults();
+ for (String each: splitter.split(line.substring("parents:".length())))
+ commitBuilderRef.get().parentHashes.add(each);
} else if (line.startsWith("author:")) {
commitBuilderRef.get().authorName = line.substring("author:".length()).trim();
} else if (line.startsWith("committer:")) {
@@ -106,48 +146,43 @@ public abstract class LogCommand extends GitCommand<Void> {
commitBuilderRef.get().authorEmail = line.substring("authorEmail:".length()).trim();
} else if (line.startsWith("committerEmail:")) {
commitBuilderRef.get().committerEmail = line.substring("committerEmail:".length()).trim();
- } else if (line.startsWith("parents:")) {
- Splitter splitter = Splitter.on(" ").omitEmptyStrings().trimResults();
- for (String each: splitter.split(line.substring("parents:".length())))
- commitBuilderRef.get().parentHashes.add(each);
} else if (line.startsWith("committerDate:")) {
commitBuilderRef.get().committerDate =
GitUtils.parseRawDate(line.substring("committerDate:".length()).trim());
} else if (line.startsWith("authorDate:")) {
commitBuilderRef.get().authorDate =
GitUtils.parseRawDate(line.substring("authorDate:".length()).trim());
- } else if (line.trim().length() != 0) {
- StringTokenizer tokenizer = new StringTokenizer(line, "\t");
- String additionsToken = tokenizer.nextToken();
- int additions = additionsToken.equals("-")?-1:Integer.parseInt(additionsToken);
- String deletionsToken = tokenizer.nextToken();
- int deletions = deletionsToken.equals("-")?-1:Integer.parseInt(deletionsToken);
-
- String path = tokenizer.nextToken();
- int renameSignIndex = path.indexOf(" => ");
- if (renameSignIndex != -1) {
- int leftBraceIndex = path.indexOf("{");
- int rightBraceIndex = path.indexOf("}");
- if (leftBraceIndex != -1 && rightBraceIndex != -1 && leftBraceIndex<renameSignIndex
- && rightBraceIndex>renameSignIndex) {
- String leftCommon = path.substring(0, leftBraceIndex);
- String rightCommon = path.substring(rightBraceIndex+1);
- String oldPath = leftCommon + path.substring(leftBraceIndex+1, renameSignIndex)
- + rightCommon;
- String newPath = leftCommon + path.substring(renameSignIndex+4, rightBraceIndex)
- + rightCommon;
- commitBuilderRef.get().fileChanges.add(
- new FileChange(oldPath, newPath, additions, deletions));
- } else {
- String oldPath = QuotedString.GIT_PATH.dequote(path.substring(0, renameSignIndex));
- String newPath = QuotedString.GIT_PATH.dequote(path.substring(renameSignIndex+4));
- commitBuilderRef.get().fileChanges.add(
- new FileChange(oldPath, newPath, additions, deletions));
- }
+ } else if (line.startsWith("commitDate:")) {
+ commitBuilderRef.get().commitDate =
+ GitUtils.parseRawDate(line.substring("commitDate:".length()).trim());
+ } else if (line.startsWith("subject:")) {
+ commitBuilderRef.get().subject = line.substring("subject:".length()).trim();
+ } else if (line.trim().length() != 0 && line.contains("\t")) {
+ FileChange change;
+ if (fields.contains(Field.LINE_CHANGES)) {
+ change = parseNumStats(line);
} else {
- path = QuotedString.GIT_PATH.dequote(path);
- commitBuilderRef.get().fileChanges.add(new FileChange(null, path, additions, deletions));
+ StringTokenizer tokenizer = new StringTokenizer(line, "\t");
+ String statusCode = tokenizer.nextToken();
+ if (statusCode.startsWith("R")) {
+ String oldPath = QuotedString.GIT_PATH.dequote(tokenizer.nextToken("\t"));
+ String newPath = QuotedString.GIT_PATH.dequote(tokenizer.nextToken("\t"));
+ change = new FileChange(oldPath, newPath, -1, -1);
+ } else if (statusCode.equals("M") || statusCode.equals("T")) {
+ String path = QuotedString.GIT_PATH.dequote(tokenizer.nextToken("\t"));
+ change = new FileChange(path, path, -1, -1);
+ } else if (statusCode.equals("D")) {
+ String oldPath = QuotedString.GIT_PATH.dequote(tokenizer.nextToken("\t"));
+ change = new FileChange(oldPath, null, -1, -1);
+ } else if (statusCode.equals("A")) {
+ String newPath = QuotedString.GIT_PATH.dequote(tokenizer.nextToken("\t"));
+ change = new FileChange(null, newPath, -1, -1);
+ } else {
+ change = null;
+ }
}
+ if (change != null)
+ commitBuilderRef.get().fileChanges.add(change);
}
}
diff --git a/server-core/src/main/java/io/onedev/server/infomanager/CommitInfoManager.java b/server-core/src/main/java/io/onedev/server/infomanager/CommitInfoManager.java
index 43c6652f85..531c49772e 100644
--- a/server-core/src/main/java/io/onedev/server/infomanager/CommitInfoManager.java
+++ b/server-core/src/main/java/io/onedev/server/infomanager/CommitInfoManager.java
@@ -17,7 +17,7 @@ public interface CommitInfoManager {
List<String> getFiles(Project project);
- int getEdits(Project project, User user, String path);
+ int getCommitCount(Project project, User user, String path);
int getCommitCount(Project project);
diff --git a/server-core/src/main/java/io/onedev/server/infomanager/DefaultCommitInfoManager.java b/server-core/src/main/java/io/onedev/server/infomanager/DefaultCommitInfoManager.java
index 4197a777f1..8a711e0955 100644
--- a/server-core/src/main/java/io/onedev/server/infomanager/DefaultCommitInfoManager.java
+++ b/server-core/src/main/java/io/onedev/server/infomanager/DefaultCommitInfoManager.java
@@ -8,18 +8,18 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
import java.util.Set;
import java.util.Stack;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
import javax.inject.Inject;
import javax.inject.Singleton;
@@ -37,12 +37,13 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import io.onedev.commons.launcher.loader.Listen;
import io.onedev.commons.launcher.loader.ListenerRegistry;
-import io.onedev.commons.utils.ExceptionUtils;
import io.onedev.commons.utils.FileUtils;
+import io.onedev.commons.utils.Pair;
import io.onedev.commons.utils.PathUtils;
import io.onedev.commons.utils.StringUtils;
import io.onedev.commons.utils.concurrent.Prioritized;
@@ -58,6 +59,7 @@ import io.onedev.server.git.GitUtils;
import io.onedev.server.git.NameAndEmail;
import io.onedev.server.git.command.FileChange;
import io.onedev.server.git.command.GitCommit;
+import io.onedev.server.git.command.ListNumStatsCommand;
import io.onedev.server.git.command.LogCommand;
import io.onedev.server.git.command.RevListCommand;
import io.onedev.server.git.command.RevListCommand.Order;
@@ -68,6 +70,7 @@ import io.onedev.server.persistence.SessionManager;
import io.onedev.server.persistence.annotation.Sessional;
import io.onedev.server.storage.StorageManager;
import io.onedev.server.util.Day;
+import io.onedev.server.util.ElementPumper;
import io.onedev.server.util.IssueUtils;
import io.onedev.server.util.work.BatchWorkManager;
import io.onedev.server.util.work.BatchWorker;
@@ -87,7 +90,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme
private static final Logger logger = LoggerFactory.getLogger(DefaultCommitInfoManager.class);
- private static final int INFO_VERSION = 7;
+ private static final int INFO_VERSION = 8;
private static final long LOG_FILE_SIZE = 256*1024;
@@ -105,7 +108,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme
private static final String FIX_COMMITS_STORE = "fixCommits";
- private static final String EDITS_STORE = "edits";
+ private static final String COMMIT_COUNTS_STORE = "commitCounts";
private static final String HISTORY_PATHS_STORE = "historyPaths";
@@ -279,27 +282,24 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme
private final SessionManager sessionManager;
- private final ExecutorService executorService;
-
private final ListenerRegistry listenerRegistry;
private final IssueManager issueManager;
private final Map<Long, List<String>> filesCache = new ConcurrentHashMap<>();
- private final Map<Long, Integer> commitCountCache = new ConcurrentHashMap<>();
+ private final Map<Long, Integer> totalCommitCountCache = new ConcurrentHashMap<>();
private final Map<Long, List<NameAndEmail>> usersCache = new ConcurrentHashMap<>();
@Inject
public DefaultCommitInfoManager(ProjectManager projectManager, StorageManager storageManager,
- BatchWorkManager batchWorkManager, SessionManager sessionManager, ExecutorService executorService,
+ BatchWorkManager batchWorkManager, SessionManager sessionManager,
ListenerRegistry listenerRegistry, IssueManager issueManager) {
this.projectManager = projectManager;
this.storageManager = storageManager;
this.batchWorkManager = batchWorkManager;
this.sessionManager = sessionManager;
- this.executorService = executorService;
this.listenerRegistry = listenerRegistry;
this.issueManager = issueManager;
}
@@ -318,285 +318,208 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme
Environment env = getEnv(project.getId().toString());
Store defaultStore = getStore(env, DEFAULT_STORE);
Store commitsStore = getStore(env, COMMITS_STORE);
- Store editsStore = getStore(env, EDITS_STORE);
+ Store commitCountsStore = getStore(env, COMMIT_COUNTS_STORE);
Store historyPathsStore = getStore(env, HISTORY_PATHS_STORE);
Store pathToIndexStore = getStore(env, PATH_TO_INDEX_STORE);
Store indexToPathStore = getStore(env, INDEX_TO_PATH_STORE);
Store emailToIndexStore = getStore(env, EMAIL_TO_INDEX_STORE);
Store indexToUserStore = getStore(env, INDEX_TO_USER_STORE);
- Store dailyContributionsStore = getStore(env, DAILY_CONTRIBUTIONS_STORE);
Store fixCommitsStore = getStore(env, FIX_COMMITS_STORE);
+ Store dailyContributionsStore = getStore(env, DAILY_CONTRIBUTIONS_STORE);
Repository repository = project.getRepository();
Collection<Long> fixedIssueNumbers = new HashSet<>();
- env.executeInTransaction(new TransactionalExecutable() {
+ Pair<byte[], ObjectId> result = env.computeInTransaction(new TransactionalComputable<Pair<byte[], ObjectId>>() {
@Override
- public void execute(Transaction txn) {
+ public Pair<byte[], ObjectId> compute(Transaction txn) {
ByteIterable commitKey = new CommitByteIterable(commitId);
byte[] commitBytes = readBytes(commitsStore, txn, commitKey);
- if (!isCommitCollected(commitBytes)) {
- int commitCount = readInt(defaultStore, txn, COMMIT_COUNT_KEY, 0);
+ ObjectId lastCommitId;
+ byte[] lastCommitBytes = readBytes(defaultStore, txn, LAST_COMMIT_KEY);
+ if (lastCommitBytes != null) {
+ lastCommitId = ObjectId.fromRaw(lastCommitBytes);
+ if (!repository.hasObject(lastCommitId))
+ lastCommitId = null;
+ } else {
+ lastCommitId = null;
+ }
+
+ return new Pair<>(commitBytes, lastCommitId);
+ }
+ });
+
+ if (!isCommitCollected(result.getFirst())) {
+ AtomicReference<ObjectId> lastCommitIdRef = new AtomicReference<>(result.getSecond());
+ RevListCommand revList = new RevListCommand(project.getGitDir());
+ List<String> revisions = new ArrayList<>();
+ revisions.add(commitId.name());
+ if (lastCommitIdRef.get() != null)
+ revisions.add("^" + lastCommitIdRef.get().name());
+ revList.revisions(revisions).order(Order.TOPO);
+
+ List<ObjectId> historyIds = new ArrayList<>();
+ for (String commitHash: revList.call())
+ historyIds.add(ObjectId.fromString(commitHash));
+
+ revList = new RevListCommand(project.getGitDir());
+ revList.revisions(revisions).order(null).firstParent(true);
+
+ Set<ObjectId> firstParentIds = new HashSet<>();
+ for (String commitHash: revList.call())
+ firstParentIds.add(ObjectId.fromString(commitHash));
+
+ /*
+ * Instead of collecting information of master branch all at once, we identify some
+ * intermediate commits and collect the information using these intermediate commits
+ * multiple times for two reasons:
+ * 1. Use less memory
+ * 2. Commit Exodus transaction sooner so user can use auto-completion when search
+ * commits even if collection is not done yet
+ */
+ List<ObjectId> intermediateCommitIds = new ArrayList<>();
+ int count = 0;
+ for (ObjectId historyId: historyIds) {
+ count++;
+ /*
+ * Only use intermediate commits that are part of first parent chain. This
+ * makes sure that subsequent intermediate commits are always ancestor of
+ * current intermediate commit (after reverse done below), to avoid
+ * collecting some commits multiple times
+ */
+ if (count > COLLECT_BATCH_SIZE && firstParentIds.contains(historyId)) {
+ intermediateCommitIds.add(historyId);
+ count = 0;
+ }
+ }
+
+ Collections.reverse(intermediateCommitIds);
+ intermediateCommitIds.add(commitId);
+
+ for(ObjectId intermediateCommitId: intermediateCommitIds) {
+ env.executeInTransaction(new TransactionalExecutable() {
- try (RevWalk revWalk = new RevWalk(project.getRepository())) {
- RevCommit commit = revWalk.lookupCommit(commitId);
- revWalk.markStart(commit);
+ @SuppressWarnings("unchecked")
+ @Override
+ public void execute(Transaction txn) {
+ AtomicInteger totalCommitCount = new AtomicInteger(readInt(defaultStore, txn, COMMIT_COUNT_KEY, 0));
- byte[] lastCommitBytes = readBytes(defaultStore, txn, LAST_COMMIT_KEY);
- if (lastCommitBytes != null) {
- ObjectId lastCommitId = ObjectId.fromRaw(lastCommitBytes);
- if (repository.hasObject(lastCommitId))
- revWalk.markUninteresting(revWalk.lookupCommit(lastCommitId));
- }
+ NextIndex nextIndex = new NextIndex();
+ nextIndex.user = readInt(defaultStore, txn, NEXT_USER_INDEX_KEY, 0);
+ nextIndex.path = readInt(defaultStore, txn, NEXT_PATH_INDEX_KEY, 0);
- RevCommit nextCommit = revWalk.next();
- while (nextCommit != null) {
- ByteIterable nextCommitKey = new CommitByteIterable(nextCommit);
- byte[] nextCommitBytes = readBytes(commitsStore, txn, nextCommitKey);
-
- if (!isCommitCollected(nextCommitBytes)) {
- byte[] newNextCommitBytes;
- if (nextCommitBytes == null) {
- newNextCommitBytes = new byte[1];
- } else {
- newNextCommitBytes = new byte[1+nextCommitBytes.length];
- System.arraycopy(nextCommitBytes, 0, newNextCommitBytes, 1, nextCommitBytes.length);
- }
-
- commitsStore.put(txn, nextCommitKey, new ArrayByteIterable(newNextCommitBytes));
-
- commitCount++;
-
- for (RevCommit parentCommit: nextCommit.getParents()) {
- ByteIterable parentCommitKey = new CommitByteIterable(parentCommit);
- byte[] parentCommitBytes = readBytes(commitsStore, txn, parentCommitKey);
- byte[] newParentCommitBytes;
- if (parentCommitBytes != null) {
- newParentCommitBytes = new byte[parentCommitBytes.length+20];
- System.arraycopy(parentCommitBytes, 0, newParentCommitBytes, 0, parentCommitBytes.length);
- } else {
- newParentCommitBytes = new byte[20];
- }
- nextCommit.copyRawTo(newParentCommitBytes, newParentCommitBytes.length-20);
- commitsStore.put(txn, parentCommitKey, new ArrayByteIterable(newParentCommitBytes));
- }
-
- for (Long issueNumber: IssueUtils.parseFixedIssueNumbers(nextCommit.getFullMessage())) {
- ByteIterable issueKey = new LongByteIterable(issueNumber);
- Collection<ObjectId> fixCommits = readCommits(fixCommitsStore, txn, issueKey);
-
- boolean addNextCommit = true;
- for (Iterator<ObjectId> it = fixCommits.iterator(); it.hasNext();) {
- ObjectId fixCommit = it.next();
- if (GitUtils.isMergedInto(project.getRepository(), null, fixCommit, nextCommit)) {
- it.remove();
- } else if (GitUtils.isMergedInto(project.getRepository(), null, nextCommit, fixCommit)) {
- addNextCommit = false;
- break;
- }
- }
- if (addNextCommit) {
- fixCommits.add(nextCommit);
- fixedIssueNumbers.add(issueNumber);
- }
- writeCommits(fixCommitsStore, txn, issueKey, fixCommits);
- }
- }
- nextCommit = revWalk.next();
- }
- writeInt(defaultStore, txn, COMMIT_COUNT_KEY, commitCount);
- commitCountCache.remove(project.getId());
+ Map<Long, Integer> commitCountCache = new HashMap<>();
- defaultStore.put(txn, LAST_COMMIT_KEY, new CommitByteIterable(commitId));
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- });
-
- for (Long issueNumber: fixedIssueNumbers) {
- Issue issue = issueManager.find(project, issueNumber);
- if (issue != null)
- listenerRegistry.post(new IssueCommitted(issue));
- }
+ Set<NameAndEmail> users;
+ byte[] userBytes = readBytes(defaultStore, txn, USERS_KEY);
+ if (userBytes != null)
+ users = (Set<NameAndEmail>) SerializationUtils.deserialize(userBytes);
+ else
+ users = new HashSet<>();
- if (GitUtils.branch2ref(project.getDefaultBranch()).equals(refName)) {
- ObjectId lastCommitId =
- env.computeInReadonlyTransaction(new TransactionalComputable<ObjectId>() {
+ Map<String, Long> files;
+ byte[] fileBytes = readBytes(defaultStore, txn, FILES_KEY);
+ if (fileBytes != null)
+ files = (Map<String, Long>) SerializationUtils.deserialize(fileBytes);
+ else
+ files = new HashMap<>();
- @Override
- public ObjectId compute(Transaction txn) {
- byte[] lastCommitBytes = readBytes(defaultStore, txn, LAST_COMMIT_OF_DEFAULT_BRANCH_KEY);
- if (lastCommitBytes != null)
- return ObjectId.fromRaw(lastCommitBytes);
- else
- return null;
- }
-
- });
+ new ElementPumper<GitCommit>() {
- if (!commitId.equals(lastCommitId)) {
- RevListCommand revList = new RevListCommand(project.getGitDir());
- List<String> revisions = new ArrayList<>();
- revisions.add(commitId.name());
- if (lastCommitId != null && repository.hasObject(lastCommitId))
- revisions.add("^" + lastCommitId.name());
- revList.revisions(revisions).order(Order.TOPO);
-
- List<ObjectId> historyIds = new ArrayList<>();
- for (String commitHash: revList.call())
- historyIds.add(ObjectId.fromString(commitHash));
-
- revList = new RevListCommand(project.getGitDir());
- revList.order(null).firstParent(true);
-
- Set<ObjectId> firstParentIds = new HashSet<>();
- for (String commitHash: revList.call())
- firstParentIds.add(ObjectId.fromString(commitHash));
+ @Override
+ public void generate(Consumer<GitCommit> consumer) {
+ List<String> revisions = new ArrayList<>();
+ revisions.add(intermediateCommitId.name());
- /*
- * Instead of collecting information of master branch all at once, we identify some
- * intermediate commits and collect the information using these intermediate commits
- * multiple times for two reasons:
- * 1. Use less memory
- * 2. Commit Exodus transaction sooner so user can use auto-completion when search
- * commits even if collection is not done yet
- */
- List<ObjectId> intermediateIds = new ArrayList<>();
- int count = 0;
- for (ObjectId historyId: historyIds) {
- count++;
- /*
- * Only use intermediate commits that are part of first parent chain. This
- * makes sure that subsequent intermediate commits are always ancestor of
- * current intermediate commit (after reverse done below), to avoid
- * collecting some commits multiple times
- */
- if (count > COLLECT_BATCH_SIZE && firstParentIds.contains(historyId)) {
- intermediateIds.add(historyId);
- count = 0;
- }
- }
+ if (lastCommitIdRef.get() != null)
+ revisions.add("^" + lastCommitIdRef.get().name());
- Collections.reverse(intermediateIds);
- intermediateIds.add(commitId);
-
- historyIds = null;
- firstParentIds = null;
-
- for(ObjectId currentCommitId: intermediateIds) {
- env.executeInTransaction(new TransactionalExecutable() {
-
- @SuppressWarnings("unchecked")
- @Override
- public void execute(Transaction txn) {
- NextIndex nextIndex = new NextIndex();
- nextIndex.user = readInt(defaultStore, txn, NEXT_USER_INDEX_KEY, 0);
- nextIndex.path = readInt(defaultStore, txn, NEXT_PATH_INDEX_KEY, 0);
-
- Map<Long, Integer> editsCache = new HashMap<>();
-
- Set<NameAndEmail> users;
- byte[] userBytes = readBytes(defaultStore, txn, USERS_KEY);
- if (userBytes != null)
- users = (Set<NameAndEmail>) SerializationUtils.deserialize(userBytes);
- else
- users = new HashSet<>();
-
- Map<String, Long> files;
- byte[] fileBytes = readBytes(defaultStore, txn, FILES_KEY);
- if (fileBytes != null)
- files = (Map<String, Long>) SerializationUtils.deserialize(fileBytes);
- else
- files = new HashMap<>();
-
- Map<Integer, Map<String, Integer>> lineStats;
- byte[] bytesOfLineStats = readBytes(defaultStore, txn, LINE_STATS_KEY);
- if (bytesOfLineStats != null) {
- lineStats = (Map<Integer, Map<String, Integer>>) SerializationUtils.deserialize(
- bytesOfLineStats);
- } else {
- lineStats = new HashMap<>();
- }
-
- Map<Integer, Map<Integer, Contribution>> dailyContributionsCache = new HashMap<>();
- Map<Integer, Contribution> overallContributions =
- deserializeContributions(readBytes(defaultStore, txn, OVERALL_CONTRIBUTIONS_KEY));
-
- /*
- * Use a synchronous queue to achieve below purpose:
- * 1. Add commit to Xodus transactional store in the same thread opening the transaction
- * as this is required by Xodus
- * 2. Do not pile up commits to use minimal memory
- */
- SynchronousQueue<Optional<GitCommit>> queue = new SynchronousQueue<>();
- AtomicReference<Exception> logException = new AtomicReference<>(null);
-
- List<String> revisions = new ArrayList<>();
- revisions.add(currentCommitId.name());
-
- ObjectId lastCommitId;
- byte[] lastCommitBytes = readBytes(defaultStore, txn, LAST_COMMIT_OF_DEFAULT_BRANCH_KEY);
- if (lastCommitBytes != null)
- lastCommitId = ObjectId.fromRaw(lastCommitBytes);
- else
- lastCommitId = null;
- if (lastCommitId != null && repository.hasObject(lastCommitId))
- revisions.add("^" + lastCommitId.name());
-
- LogCommand log = new LogCommand(project.getGitDir()) {
-
- @Override
- protected void consume(GitCommit commit) {
- try {
- queue.put(Optional.of(commit));
- } catch (InterruptedException e) {
+ EnumSet<LogCommand.Field> fields = EnumSet.allOf(LogCommand.Field.class);
+ fields.remove(LogCommand.Field.LINE_CHANGES);
+ new LogCommand(project.getGitDir()) {
+
+ @Override
+ protected void consume(GitCommit commit) {
+ consumer.accept(commit);
}
- }
+
+ }.revisions(revisions).fields(fields).call();
+ }
+
+ @Override
+ public void process(GitCommit currentCommit) {
+ ObjectId currentCommitId = ObjectId.fromString(currentCommit.getHash());
+ ByteIterable currentCommitKey = new CommitByteIterable(currentCommitId);
+ byte[] currentCommitBytes = readBytes(commitsStore, txn, currentCommitKey);
- }.revisions(revisions);
-
- executorService.execute(new Runnable() {
-
- @Override
- public void run() {
- try {
- log.call();
- } catch (Exception e) {
- logException.set(e);
- } finally {
- try {
- queue.put(Optional.empty());
- } catch (InterruptedException e) {
+ if (!isCommitCollected(currentCommitBytes)) {
+ totalCommitCount.incrementAndGet();
+
+ byte[] newCurrentCommitBytes;
+ if (currentCommitBytes == null) {
+ newCurrentCommitBytes = new byte[1];
+ } else {
+ newCurrentCommitBytes = new byte[1+currentCommitBytes.length];
+ System.arraycopy(currentCommitBytes, 0, newCurrentCommitBytes, 1, currentCommitBytes.length);
+ }
+
+ commitsStore.put(txn, currentCommitKey, new ArrayByteIterable(newCurrentCommitBytes));
+
+ for (String parentCommitHash: currentCommit.getParentHashes()) {
+ ByteIterable parentCommitKey = new CommitByteIterable(ObjectId.fromString(parentCommitHash));
+ byte[] parentCommitBytes = readBytes(commitsStore, txn, parentCommitKey);
+ byte[] newParentCommitBytes;
+ if (parentCommitBytes != null) {
+ newParentCommitBytes = new byte[parentCommitBytes.length+20];
+ System.arraycopy(parentCommitBytes, 0, newParentCommitBytes, 0, parentCommitBytes.length);
+ } else {
+ newParentCommitBytes = new byte[20];
}
+ currentCommitId.copyRawTo(newParentCommitBytes, newParentCommitBytes.length-20);
+ commitsStore.put(txn, parentCommitKey, new ArrayByteIterable(newParentCommitBytes));
+ }
+
+ String commitMessage = currentCommit.getSubject();
+ if (currentCommit.getBody() != null)
+ commitMessage += "\n\n" + currentCommit.getBody();
+
+ for (Long issueNumber: IssueUtils.parseFixedIssueNumbers(commitMessage)) {
+ ByteIterable issueKey = new LongByteIterable(issueNumber);
+ Collection<ObjectId> fixCommits = readCommits(fixCommitsStore, txn, issueKey);
+
+ boolean addNextCommit = true;
+ for (Iterator<ObjectId> it = fixCommits.iterator(); it.hasNext();) {
+ ObjectId fixCommit = it.next();
+ if (GitUtils.isMergedInto(project.getRepository(), null, fixCommit, currentCommitId)) {
+ it.remove();
+ } else if (GitUtils.isMergedInto(project.getRepository(), null, currentCommitId, fixCommit)) {
+ addNextCommit = false;
+ break;
+ }
+ }
+ if (addNextCommit) {
+ fixCommits.add(currentCommitId);
+ fixedIssueNumbers.add(issueNumber);
+ }
+ writeCommits(fixCommitsStore, txn, issueKey, fixCommits);
}
- }
-
- });
-
- try {
- Optional<GitCommit> logCommitOptional = queue.take();
- while (logCommitOptional.isPresent()) {
- GitCommit logCommit = logCommitOptional.get();
- if (logCommit.getCommitDate() != null) {
- for (String file: logCommit.getChangedFiles())
- files.put(file, logCommit.getCommitDate().getTime());
+ if (currentCommit.getCommitDate() != null) {
+ for (String file: currentCommit.getChangedFiles())
+ files.put(file, currentCommit.getCommitDate().getTime());
}
- if (logCommit.getCommitter() != null)
- users.add(new NameAndEmail(logCommit.getCommitter()));
+ if (currentCommit.getCommitter() != null)
+ users.add(new NameAndEmail(currentCommit.getCommitter()));
- if (logCommit.getAuthor() != null) {
- NameAndEmail nameAndEmail = new NameAndEmail(logCommit.getAuthor());
+ if (currentCommit.getAuthor() != null) {
+ NameAndEmail nameAndEmail = new NameAndEmail(currentCommit.getAuthor());
users.add(nameAndEmail);
- String emailAddress = logCommit.getAuthor().getEmailAddress();
+ String emailAddress = currentCommit.getAuthor().getEmailAddress();
if (StringUtils.isNotBlank(emailAddress)) {
ByteIterable emailKey = new StringByteIterable(emailAddress);
int userIndex = readInt(emailToIndexStore, txn, emailKey, -1);
@@ -608,48 +531,30 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme
new ArrayByteIterable(SerializationUtils.serialize(nameAndEmail)));
}
- if (logCommit.getCommitDate() != null && logCommit.getParentHashes().size() <= 1) {
- int dayValue = new Day(logCommit.getCommitDate()).getValue();
- Map<Integer, Contribution> contributionsOnDay =
- dailyContributionsCache.get(dayValue);
- if (contributionsOnDay == null) {
- contributionsOnDay = deserializeContributions(readBytes(
- dailyContributionsStore, txn, new IntByteIterable(dayValue)));
- dailyContributionsCache.put(dayValue, contributionsOnDay);
- }
- updateContribution(contributionsOnDay, userIndex, logCommit);
- }
-
- for (FileChange change: logCommit.getFileChanges()) {
- String path = change.getPath();
- int pathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,
- nextIndex, path);
- int edits = change.getAdditions() + change.getDeletions();
- if (edits < 0)
- edits = 100;
- updateEdits(editsStore, txn, editsCache, userIndex, pathIndex, edits);
- while (path.contains("/")) {
- path = StringUtils.substringBeforeLast(path, "/");
- pathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,
+ for (FileChange change: currentCommit.getFileChanges()) {
+ for (String path: change.getPaths()) {
+ int pathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,
nextIndex, path);
- updateEdits(editsStore, txn, editsCache, userIndex, pathIndex, edits);
+ updateCommitCount(commitCountsStore, txn, commitCountCache, userIndex, pathIndex);
+ while (path.contains("/")) {
+ path = StringUtils.substringBeforeLast(path, "/");
+ pathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,
+ nextIndex, path);
+ updateCommitCount(commitCountsStore, txn, commitCountCache, userIndex, pathIndex);
+ }
+ pathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,
+ nextIndex, "");
+ updateCommitCount(commitCountsStore, txn, commitCountCache, userIndex, pathIndex);
}
- pathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,
- nextIndex, "");
- updateEdits(editsStore, txn, editsCache, userIndex, pathIndex, edits);
}
}
}
- if (logCommit.getCommitDate() != null && logCommit.getParentHashes().size() <= 1) {
- int dayValue = new Day(logCommit.getCommitDate()).getValue();
- updateContribution(overallContributions, dayValue, logCommit);
- }
-
- for (FileChange change: logCommit.getFileChanges()) {
- if (change.getOldPath() != null) {
+ for (FileChange change: currentCommit.getFileChanges()) {
+ if (change.getOldPath() != null && change.getNewPath() != null
+ && !change.getOldPath().equals(change.getNewPath())) {
int pathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,
- nextIndex, change.getPath());
+ nextIndex, change.getNewPath());
ByteIterable pathKey = new IntByteIterable(pathIndex);
Set<Integer> historyPathIndexes = new HashSet<>();
byte[] bytesOfHistoryPaths = readBytes(historyPathsStore, txn, pathKey);
@@ -680,103 +585,227 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme
}
}
}
- }
-
- if (logCommit.getCommitDate() != null && logCommit.getParentHashes().size() <= 1) {
- int day = new Day(logCommit.getCommitDate()).getValue();
-
- Map<String, Integer> lineStatsOnDay = lineStats.get(day);
- if (lineStatsOnDay == null) {
- lineStatsOnDay = new HashMap<>();
- lineStats.put(day, lineStatsOnDay);
- }
-
- Map<String, Integer> languageLines = new HashMap<>();
- for (FileChange change: logCommit.getFileChanges()) {
- int lines = change.getAdditions() - change.getDeletions();
- int lastIndexOfDot = change.getPath().lastIndexOf('.');
- if (lastIndexOfDot != -1 && lines != 0) {
- String fileExt = change.getPath().substring(lastIndexOfDot+1).toLowerCase();
- String language = PROGRAMMING_LANGUAGES.get(fileExt);
- if (language != null) {
- Integer accumulatedLines = languageLines.get(language);
- if (accumulatedLines != null)
- lines += accumulatedLines;
- languageLines.put(language, lines);
- }
- }
- }
-
- for (Map.Entry<String, Integer> entry: languageLines.entrySet()) {
- String language = entry.getKey();
- Integer lines = entry.getValue();
- Integer accumulatedLines = lineStatsOnDay.get(language);
- if (accumulatedLines != null)
- lines += accumulatedLines;
- lineStatsOnDay.put(language, lines);
- }
+ }
+ }
+ }
+ }.pump();
+
+ writeInt(defaultStore, txn, COMMIT_COUNT_KEY, totalCommitCount.get());
+ totalCommitCountCache.remove(project.getId());
+
+ writeInt(defaultStore, txn, NEXT_USER_INDEX_KEY, nextIndex.user);
+ writeInt(defaultStore, txn, NEXT_PATH_INDEX_KEY, nextIndex.path);
+
+ userBytes = SerializationUtils.serialize((Serializable) users);
+ defaultStore.put(txn, USERS_KEY, new ArrayByteIterable(userBytes));
+ usersCache.remove(project.getId());
+
+ if (files.size() > MAX_COLLECTING_FILES) {
+ List<String> fileList = new ArrayList<>(files.keySet());
+ fileList.sort((file1, file2)->files.get(file1).compareTo(files.get(file2)));
+ for (int i=0; i<fileList.size() - MAX_COLLECTING_FILES; i++)
+ files.remove(fileList.get(i));
+ }
+ fileBytes = SerializationUtils.serialize((Serializable) files);
+ defaultStore.put(txn, FILES_KEY, new ArrayByteIterable(fileBytes));
+ filesCache.remove(project.getId());
+
+ for (Map.Entry<Long, Integer> entry: commitCountCache.entrySet())
+ writeInt(commitCountsStore, txn, new LongByteIterable(entry.getKey()), entry.getValue());
+
+ lastCommitIdRef.set(intermediateCommitId);
+ defaultStore.put(txn, LAST_COMMIT_KEY, new CommitByteIterable(lastCommitIdRef.get()));
+ }
+ });
+ }
+ }
+
+ if (GitUtils.branch2ref(project.getDefaultBranch()).equals(refName)) {
+ ObjectId lastCommitId = env.computeInTransaction(new TransactionalComputable<ObjectId>() {
+
+ @Override
+ public ObjectId compute(Transaction txn) {
+ byte[] lastCommitBytes = readBytes(defaultStore, txn, LAST_COMMIT_OF_DEFAULT_BRANCH_KEY);
+ if (lastCommitBytes != null) {
+ ObjectId lastCommitId = ObjectId.fromRaw(lastCommitBytes);
+ if (repository.hasObject(lastCommitId)
+ && GitUtils.isMergedInto(repository, null, lastCommitId, commitId)) {
+ return lastCommitId;
+ }
+ }
+ return null;
+ }
+
+ });
+
+ if (lastCommitId == null) {
+ Map<Integer, Map<String, Integer>> lineStats = new HashMap<>();
+ Map<Integer, Map<Integer, Contribution>> dailyContributionsCache = new HashMap<>();
+ Map<Integer, Contribution> overallContributions = new HashMap<>();
+
+ env.executeInTransaction(new TransactionalExecutable() {
+
+ @Override
+ public void execute(Transaction txn) {
+ new ElementPumper<GitCommit>() {
+
+ @Override
+ public void generate(Consumer<GitCommit> consumer) {
+ List<String> revisions = new ArrayList<>();
+ revisions.add(commitId.name());
+
+ EnumSet<LogCommand.Field> fields = EnumSet.allOf(LogCommand.Field.class);
+ fields.remove(LogCommand.Field.SUBJECT);
+ fields.remove(LogCommand.Field.BODY);
+ fields.remove(LogCommand.Field.COMMITTER);
+ fields.remove(LogCommand.Field.PARENTS);
+ new LogCommand(project.getGitDir()) {
+
+ @Override
+ protected void consume(GitCommit commit) {
+ consumer.accept(commit);
}
- logCommitOptional = queue.take();
- }
- if (logException.get() != null)
- throw logException.get();
- } catch (Exception e) {
- throw ExceptionUtils.unchecked(e);
- }
-
- for (Map.Entry<Integer, Map<Integer, Contribution>> entry: dailyContributionsCache.entrySet()) {
- byte[] bytesOfContributionsOnDay = serializeContributions(entry.getValue());
- dailyContributionsStore.put(txn, new IntByteIterable(entry.getKey()),
- new ArrayByteIterable(bytesOfContributionsOnDay));
+ }.firstParent(true).revisions(revisions).fields(fields).call();
}
- defaultStore.put(txn, OVERALL_CONTRIBUTIONS_KEY,
- new ArrayByteIterable(serializeContributions(overallContributions)));
-
- bytesOfLineStats = SerializationUtils.serialize((Serializable) lineStats);
- defaultStore.put(txn, LINE_STATS_KEY, new ArrayByteIterable(bytesOfLineStats));
-
- writeInt(defaultStore, txn, NEXT_USER_INDEX_KEY, nextIndex.user);
- writeInt(defaultStore, txn, NEXT_PATH_INDEX_KEY, nextIndex.path);
-
- userBytes = SerializationUtils.serialize((Serializable) users);
- defaultStore.put(txn, USERS_KEY, new ArrayByteIterable(userBytes));
- usersCache.remove(project.getId());
-
- if (files.size() > MAX_COLLECTING_FILES) {
- List<String> fileList = new ArrayList<>(files.keySet());
- fileList.sort((file1, file2)->files.get(file1).compareTo(files.get(file2)));
- for (int i=0; i<fileList.size() - MAX_COLLECTING_FILES; i++)
- files.remove(fileList.get(i));
+
+ @Override
+ public void process(GitCommit currentCommit) {
+ updateOverallContributionsAndLineStats(txn, emailToIndexStore,
+ currentCommit, lineStats, overallContributions);
+ int dayValue = new Day(currentCommit.getCommitDate()).getValue();
+ Map<Integer, Contribution> contributionsOnDay = dailyContributionsCache.get(dayValue);
+ if (contributionsOnDay == null) {
+ contributionsOnDay = new HashMap<>();
+ dailyContributionsCache.put(dayValue, contributionsOnDay);
+ }
+ updateContribution(txn, emailToIndexStore, currentCommit, contributionsOnDay);
}
- fileBytes = SerializationUtils.serialize((Serializable) files);
- defaultStore.put(txn, FILES_KEY, new ArrayByteIterable(fileBytes));
- filesCache.remove(project.getId());
-
- for (Map.Entry<Long, Integer> entry: editsCache.entrySet())
- writeInt(editsStore, txn, new LongByteIterable(entry.getKey()), entry.getValue());
- defaultStore.put(txn, LAST_COMMIT_OF_DEFAULT_BRANCH_KEY, new CommitByteIterable(currentCommitId));
+ }.pump();
+
+ for (int dayValue: deserializeContributions(readBytes(defaultStore, txn, OVERALL_CONTRIBUTIONS_KEY)).keySet())
+ dailyContributionsStore.delete(txn, new IntByteIterable(dayValue));
+ for (Map.Entry<Integer, Map<Integer, Contribution>> entry: dailyContributionsCache.entrySet()) {
+ byte[] bytesOfContributionsOnDay = serializeContributions(entry.getValue());
+ dailyContributionsStore.put(txn, new IntByteIterable(entry.getKey()),
+ new ArrayByteIterable(bytesOfContributionsOnDay));
}
- });
- }
- }
+ defaultStore.put(txn, OVERALL_CONTRIBUTIONS_KEY,
+ new ArrayByteIterable(serializeContributions(overallContributions)));
+
+ byte[] bytesOfLineStats = SerializationUtils.serialize((Serializable) lineStats);
+ defaultStore.put(txn, LINE_STATS_KEY, new ArrayByteIterable(bytesOfLineStats));
+
+ defaultStore.put(txn, LAST_COMMIT_OF_DEFAULT_BRANCH_KEY, new CommitByteIterable(commitId));
+ }
+
+ });
+ } else {
+ env.executeInTransaction(new TransactionalExecutable() {
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void execute(Transaction txn) {
+ Map<Integer, Map<String, Integer>> lineStats;
+ byte[] bytesOfLineStats = readBytes(defaultStore, txn, LINE_STATS_KEY);
+ if (bytesOfLineStats != null) {
+ lineStats = (Map<Integer, Map<String, Integer>>) SerializationUtils.deserialize(
+ bytesOfLineStats);
+ } else {
+ lineStats = new HashMap<>();
+ }
+
+ ListNumStatsCommand command = new ListNumStatsCommand(project.getGitDir());
+ List<FileChange> fileChanges = command.fromRev(lastCommitId.name()).toRev(commitId.name()).call();
+ RevCommit revCommit = project.getRevCommit(commitId, true);
+ GitCommit gitCommit = new GitCommit(revCommit.name(), null, null, revCommit.getAuthorIdent(),
+ revCommit.getCommitterIdent().getWhen(), null, null, fileChanges);
+
+ int dayValue = new Day(revCommit.getCommitterIdent().getWhen()).getValue();
+ Map<Integer, Contribution> contributionsOnDay = deserializeContributions(
+ readBytes(dailyContributionsStore, txn, new IntByteIterable(dayValue)));
+ Map<Integer, Contribution> overallContributions =
+ deserializeContributions(readBytes(defaultStore, txn, OVERALL_CONTRIBUTIONS_KEY));
+
+ updateOverallContributionsAndLineStats(txn, emailToIndexStore, gitCommit, lineStats, overallContributions);
+ updateContribution(txn, emailToIndexStore, gitCommit, contributionsOnDay);
+
+ byte[] bytesOfContributionsOnDay = serializeContributions(contributionsOnDay);
+ dailyContributionsStore.put(txn, new IntByteIterable(dayValue),
+ new ArrayByteIterable(bytesOfContributionsOnDay));
+ defaultStore.put(txn, OVERALL_CONTRIBUTIONS_KEY,
+ new ArrayByteIterable(serializeContributions(overallContributions)));
+
+ bytesOfLineStats = SerializationUtils.serialize((Serializable) lineStats);
+ defaultStore.put(txn, LINE_STATS_KEY, new ArrayByteIterable(bytesOfLineStats));
+
+ defaultStore.put(txn, LAST_COMMIT_OF_DEFAULT_BRANCH_KEY, new CommitByteIterable(commitId));
+ }
+
+ });
+
+ }
+ }
+
+ for (Long issueNumber: fixedIssueNumbers) {
+ Issue issue = issueManager.find(project, issueNumber);
+ if (issue != null)
+ listenerRegistry.post(new IssueCommitted(issue));
}
+
logger.debug("Collected commit information (project: {}, ref: {})", project.getName(), refName);
}
- private void updateContribution(Map<Integer, Contribution> contributions, int key, GitCommit commit) {
- Contribution contribution = contributions.get(key);
- if (contribution != null) {
- contribution = new Contribution(
- contribution.getCommits()+1,
- contribution.getAdditions()+commit.getAdditions(),
- contribution.getDeletions()+commit.getDeletions());
- } else {
- contribution = new Contribution(1, commit.getAdditions(), commit.getDeletions());
+ private void updateContribution(Transaction txn, Store emailToIndexStore,
+ GitCommit currentCommit, Map<Integer, Contribution> contributionsOnDay) {
+ if (currentCommit.getAuthor() != null
+ && StringUtils.isNotBlank(currentCommit.getAuthor().getEmailAddress())) {
+ String emailAddress = currentCommit.getAuthor().getEmailAddress();
+ ByteIterable emailKey = new StringByteIterable(emailAddress);
+ int userIndex = readInt(emailToIndexStore, txn, emailKey, -1);
+ Preconditions.checkState(userIndex != -1);
+ updateContribution(contributionsOnDay, userIndex, currentCommit);
+ }
+ }
+
+ private void updateOverallContributionsAndLineStats(Transaction txn, Store emailToIndexStore,
+ GitCommit currentCommit, Map<Integer, Map<String, Integer>> lineStats,
+ Map<Integer, Contribution> overallContributions) {
+ int dayValue = new Day(currentCommit.getCommitDate()).getValue();
+ updateContribution(overallContributions, dayValue, currentCommit);
+
+ Map<String, Integer> lineStatsOnDay = lineStats.get(dayValue);
+ if (lineStatsOnDay == null) {
+ lineStatsOnDay = new HashMap<>();
+ lineStats.put(dayValue, lineStatsOnDay);
+ }
+
+ Map<String, Integer> languageLines = new HashMap<>();
+ for (FileChange change: currentCommit.getFileChanges()) {
+ int lines = change.getAdditions() - change.getDeletions();
+ int lastIndexOfDot = change.getNewPath().lastIndexOf('.');
+ if (lastIndexOfDot != -1 && lines != 0) {
+ String fileExt = change.getNewPath().substring(lastIndexOfDot+1).toLowerCase();
+ String language = PROGRAMMING_LANGUAGES.get(fileExt);
+ if (language != null) {
+ Integer accumulatedLines = languageLines.get(language);
+ if (accumulatedLines != null)
+ lines += accumulatedLines;
+ languageLines.put(language, lines);
+ }
+ }
+ }
+
+ for (Map.Entry<String, Integer> entry: languageLines.entrySet()) {
+ String language = entry.getKey();
+ Integer lines = entry.getValue();
+ Integer accumulatedLines = lineStatsOnDay.get(language);
+ if (accumulatedLines != null)
+ lines += accumulatedLines;
+ lineStatsOnDay.put(language, lines);
}
- contributions.put(key, contribution);
}
private int getPathIndex(Store pathToIndexStore, Store indexToPathStore, Transaction txn,
@@ -791,15 +820,15 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme
return pathIndex;
}
- private void updateEdits(Store store, Transaction txn,
- Map<Long, Integer> editsCache, int userIndex, int pathIndex, int edits) {
- long editsKey = (userIndex<<32)|pathIndex;
+ private void updateCommitCount(Store store, Transaction txn,
+ Map<Long, Integer> commitCountCache, int userIndex, int pathIndex) {
+ long commitCountKey = (userIndex<<32)|pathIndex;
- Integer editsOfPathByUser = editsCache.get(editsKey);
- if (editsOfPathByUser == null)
- editsOfPathByUser = readInt(store, txn, new LongByteIterable(editsKey), 0);
- editsOfPathByUser += edits;
- editsCache.put(editsKey, editsOfPathByUser);
+ Integer commitCountOfPathByUser = commitCountCache.get(commitCountKey);
+ if (commitCountOfPathByUser == null)
+ commitCountOfPathByUser = readInt(store, txn, new LongByteIterable(commitCountKey), 0);
+ commitCountOfPathByUser ++;
+ commitCountCache.put(commitCountKey, commitCountOfPathByUser);
}
@Override
@@ -895,13 +924,26 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme
});
}
+ private void updateContribution(Map<Integer, Contribution> contributions, int key, GitCommit commit) {
+ Contribution contribution = contributions.get(key);
+ if (contribution != null) {
+ contribution = new Contribution(
+ contribution.getCommits()+1,
+ contribution.getAdditions()+commit.getAdditions(),
+ contribution.getDeletions()+commit.getDeletions());
+ } else {
+ contribution = new Contribution(1, commit.getAdditions(), commit.getDeletions());
+ }
+ contributions.put(key, contribution);
+ }
+
@Override
- public int getEdits(Project project, User user, String path) {
+ public int getCommitCount(Project project, User user, String path) {
if (user.getEmail() != null) {
Environment env = getEnv(project.getId().toString());
Store emailToIndexStore = getStore(env, EMAIL_TO_INDEX_STORE);
Store pathToIndexStore = getStore(env, PATH_TO_INDEX_STORE);
- Store editsStore = getStore(env, EDITS_STORE);
+ Store commitCountStore = getStore(env, COMMIT_COUNTS_STORE);
return env.computeInReadonlyTransaction(new TransactionalComputable<Integer>() {
@Override
@@ -910,8 +952,8 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme
if (userIndex != -1) {
int pathIndex = readInt(pathToIndexStore, txn, new StringByteIterable(path), -1);
if (pathIndex != -1) {
- long editsKey = (userIndex<<32)|pathIndex;
- return readInt(editsStore, txn, new LongByteIterable(editsKey), 0);
+ long commitCountKey = (userIndex<<32)|pathIndex;
+ return readInt(commitCountStore, txn, new LongByteIterable(commitCountKey), 0);
}
}
return 0;
@@ -973,7 +1015,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme
Long projectId = event.getEntity().getId();
removeEnv(projectId.toString());
filesCache.remove(projectId);
- commitCountCache.remove(projectId);
+ totalCommitCountCache.remove(projectId);
usersCache.remove(projectId);
}
}
@@ -1053,7 +1095,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme
@Sessional
@Override
public int getCommitCount(Project project) {
- Integer commitCount = commitCountCache.get(project.getId());
+ Integer commitCount = totalCommitCountCache.get(project.getId());
if (commitCount == null) {
Environment env = getEnv(project.getId().toString());
Store store = getStore(env, DEFAULT_STORE);
@@ -1065,7 +1107,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme
return readInt(store, txn, COMMIT_COUNT_KEY, 0);
}
});
- commitCountCache.put(project.getId(), commitCount);
+ totalCommitCountCache.put(project.getId(), commitCount);
}
return commitCount;
}
@@ -1356,12 +1398,6 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme
return INFO_VERSION;
}
- private static class NextIndex {
- int user;
-
- int path;
- }
-
@Override
public Collection<ObjectId> getFixCommits(Project project, Long issueNumber) {
Environment env = getEnv(project.getId().toString());
@@ -1378,4 +1414,10 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme
}
+ private static class NextIndex {
+ int user;
+
+ int path;
+ }
+
}
diff --git a/server-core/src/main/java/io/onedev/server/maintenance/DefaultDataManager.java b/server-core/src/main/java/io/onedev/server/maintenance/DefaultDataManager.java
index 7ea0f42f0d..3a3ad9c561 100644
--- a/server-core/src/main/java/io/onedev/server/maintenance/DefaultDataManager.java
+++ b/server-core/src/main/java/io/onedev/server/maintenance/DefaultDataManager.java
@@ -163,7 +163,7 @@ public class DefaultDataManager implements DataManager, Serializable {
setting = settingManager.getSetting(Key.JOB_EXECUTORS);
if (setting == null) {
AutoDiscoveredJobExecutor executor = new AutoDiscoveredJobExecutor();
- executor.setName("auto discovered");
+ executor.setName("auto-discovered");
settingManager.saveJobExecutors(Lists.newArrayList(executor));
}
setting = settingManager.getSetting(Key.JOB_SCRIPTS);
diff --git a/server-core/src/main/java/io/onedev/server/model/Project.java b/server-core/src/main/java/io/onedev/server/model/Project.java
index 019cfb2654..3bb4b3886a 100644
--- a/server-core/src/main/java/io/onedev/server/model/Project.java
+++ b/server-core/src/main/java/io/onedev/server/model/Project.java
@@ -557,31 +557,38 @@ public class Project extends AbstractEntity {
Optional<Blob> blob = getBlobCache().get(blobIdent);
if (blob == null) {
try (RevWalk revWalk = new RevWalk(getRepository())) {
- ObjectId revId = getObjectId(blobIdent.revision, true);
- RevTree revTree = revWalk.parseCommit(revId).getTree();
- TreeWalk treeWalk = TreeWalk.forPath(getRepository(), blobIdent.path, revTree);
- if (treeWalk != null) {
- ObjectId blobId = treeWalk.getObjectId(0);
- if (blobIdent.isGitLink()) {
- String url = getSubmodules(blobIdent.revision).get(blobIdent.path);
- if (url == null) {
- if (mustExist)
- throw new ObjectNotFoundException("Unable to find submodule '" + blobIdent.path + "' in .gitmodules");
- else
- blob = Optional.absent();
- } else {
- String hash = blobId.name();
- blob = Optional.of(new Blob(blobIdent, blobId, new Submodule(url, hash).toString().getBytes()));
- }
- } else if (blobIdent.isTree()) {
- throw new NotFileException("Path '" + blobIdent.path + "' is a tree");
- } else {
- blob = Optional.of(new Blob(blobIdent, blobId, treeWalk.getObjectReader()));
- }
- } else if (mustExist) {
- throw new ObjectNotFoundException("Unable to find blob path '" + blobIdent.path + "' in revision '" + blobIdent.revision + "'");
- } else {
- blob = Optional.absent();
+ ObjectId revId = getObjectId(blobIdent.revision, mustExist);
+ if (revId != null) {
+ RevCommit commit = GitUtils.parseCommit(revWalk, revId);
+ if (commit != null) {
+ RevTree revTree = commit.getTree();
+ TreeWalk treeWalk = TreeWalk.forPath(getRepository(), blobIdent.path, revTree);
+ if (treeWalk != null) {
+ ObjectId blobId = treeWalk.getObjectId(0);
+ if (blobIdent.isGitLink()) {
+ String url = getSubmodules(blobIdent.revision).get(blobIdent.path);
+ if (url == null) {
+ if (mustExist)
+ throw new ObjectNotFoundException("Unable to find submodule '" + blobIdent.path + "' in .gitmodules");
+ else
+ blob = Optional.absent();
+ } else {
+ String hash = blobId.name();
+ blob = Optional.of(new Blob(blobIdent, blobId, new Submodule(url, hash).toString().getBytes()));
+ }
+ } else if (blobIdent.isTree()) {
+ throw new NotFileException("Path '" + blobIdent.path + "' is a tree");
+ } else {
+ blob = Optional.of(new Blob(blobIdent, blobId, treeWalk.getObjectReader()));
+ }
+ }
+ }
+ }
+ if (blob == null) {
+ if (mustExist)
+ throw new ObjectNotFoundException("Unable to find blob ident: " + blobIdent);
+ else
+ blob = Optional.absent();
}
getBlobCache().put(blobIdent, blob);
} catch (IOException e) {
diff --git a/server-core/src/main/java/io/onedev/server/model/PullRequestUpdate.java b/server-core/src/main/java/io/onedev/server/model/PullRequestUpdate.java
index 9072627dc6..f245f186c1 100644
--- a/server-core/src/main/java/io/onedev/server/model/PullRequestUpdate.java
+++ b/server-core/src/main/java/io/onedev/server/model/PullRequestUpdate.java
@@ -31,8 +31,6 @@ import com.google.common.base.Preconditions;
import io.onedev.server.entitymanager.impl.DefaultCodeCommentRelationManager;
import io.onedev.server.git.GitUtils;
-import io.onedev.server.git.command.FileChange;
-import io.onedev.server.git.command.ListFileChangesCommand;
@Entity
@Table(indexes={@Index(columnList="o_request_id"), @Index(columnList="date")})
@@ -59,8 +57,6 @@ public class PullRequestUpdate extends AbstractEntity {
private transient Collection<String> changedFiles;
- private transient Collection<FileChange> fileChanges;
-
public PullRequest getRequest() {
return request;
}
@@ -103,16 +99,6 @@ public class PullRequestUpdate extends AbstractEntity {
GitUtils.deleteRef(GitUtils.getRefUpdate(getRequest().getTargetProject().getRepository(), getHeadRef()));
}
- public Collection<FileChange> getFileChanges() {
- if (fileChanges == null) {
- ListFileChangesCommand cmd = new ListFileChangesCommand(getRequest().getTargetProject().getGitDir());
- cmd.fromRev(getBaseCommitHash());
- cmd.toRev(getHeadCommitHash());
- fileChanges = cmd.call();
- }
- return fileChanges;
- }
-
/**
* Get changed files of this update since previous update. This calculation
* excludes changes introduced by commits from target branch (this will
diff --git a/server-core/src/main/java/io/onedev/server/notification/CommitNotificationManager.java b/server-core/src/main/java/io/onedev/server/notification/CommitNotificationManager.java
index 012ef2c2c7..928ef10122 100644
--- a/server-core/src/main/java/io/onedev/server/notification/CommitNotificationManager.java
+++ b/server-core/src/main/java/io/onedev/server/notification/CommitNotificationManager.java
@@ -84,11 +84,13 @@ public class CommitNotificationManager {
}
}
- RevCommit commit = project.getRevCommit(event.getNewCommitId(), true);
- String subject = String.format("Subscribed commit at ref '%s': %s", event.getRefName(), commit.getShortMessage());
- String url = urlManager.urlFor(project, commit);
- String body = String.format("Visit <a href='%s'>%s</a> for details", url, url);
- mailManager.sendMailAsync(notifyEmails, subject, body.toString());
+ RevCommit commit = project.getRevCommit(event.getNewCommitId(), false);
+ if (commit != null) {
+ String subject = String.format("Subscribed commit at ref '%s': %s", event.getRefName(), commit.getShortMessage());
+ String url = urlManager.urlFor(project, commit);
+ String body = String.format("Visit <a href='%s'>%s</a> for details", url, url);
+ mailManager.sendMailAsync(notifyEmails, subject, body.toString());
+ }
}
}
}
diff --git a/server-core/src/main/java/io/onedev/server/util/ElementPumper.java b/server-core/src/main/java/io/onedev/server/util/ElementPumper.java
new file mode 100644
index 0000000000..338fdf45e2
--- /dev/null
+++ b/server-core/src/main/java/io/onedev/server/util/ElementPumper.java
@@ -0,0 +1,74 @@
+package io.onedev.server.util;
+
+import java.util.Optional;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+
+import io.onedev.commons.utils.ExceptionUtils;
+import io.onedev.server.OneDev;
+
+/**
+ * A pumper generates elements in a different thread, and consumes the element
+ * in current thread. It is designed as some scenario requires that elements
+ * should be consumed in current thread, and collecting all generated elements
+ * into a collection before processing is not acceptable due to memory constraint
+ *
+ * @author robin
+ *
+ * @param <T>
+ */
+public abstract class ElementPumper<T> {
+
+ public void pump() {
+ SynchronousQueue<Optional<T>> queue = new SynchronousQueue<>();
+ AtomicReference<Exception> exceptionRef = new AtomicReference<>(null);
+
+ OneDev.getInstance(ExecutorService.class).execute(new Runnable() {
+
+ @Override
+ public void run() {
+ try {
+ generate(new Consumer<T>() {
+
+ @Override
+ public void accept(T t) {
+ try {
+ queue.put(Optional.of(t));
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ });
+ } catch (Exception e) {
+ exceptionRef.set(e);
+ } finally {
+ try {
+ queue.put(Optional.empty());
+ } catch (InterruptedException e) {
+ }
+ }
+ }
+
+ });
+
+ try {
+ Optional<T> element = queue.take();
+ while (element.isPresent()) {
+ process(element.get());
+ element = queue.take();
+ }
+ if (exceptionRef.get() != null)
+ throw exceptionRef.get();
+ } catch (Exception e) {
+ throw ExceptionUtils.unchecked(e);
+ }
+ }
+
+ public abstract void generate(Consumer<T> consumer);
+
+ public abstract void process(T element);
+
+}
diff --git a/server-core/src/main/java/io/onedev/server/web/component/diff/revision/RevisionDiffPanel.html b/server-core/src/main/java/io/onedev/server/web/component/diff/revision/RevisionDiffPanel.html
index fda557ffa4..fc716ead70 100644
--- a/server-core/src/main/java/io/onedev/server/web/component/diff/revision/RevisionDiffPanel.html
+++ b/server-core/src/main/java/io/onedev/server/web/component/diff/revision/RevisionDiffPanel.html
@@ -15,10 +15,10 @@
<td class="whitespace-option">
<span wicket:id="whitespaceOption" class="btn btn-default" title="White space options">White space <i class="caret"></i></span>
</td>
- <td class="path-filter">
- <form wicket:id="pathFilter">
+ <td class="path-filter">
+ <form wicket:id="pathFilter" class="clearable-wrapper">
<div class="input-group">
- <input wicket:id="input" class="form-control no-autofocus" spellcheck="false" autocomplete="off" placeholder="Filter by path" type="text">
+ <input wicket:id="input" class="form-control no-autofocus" spellcheck="false" autocomplete="off" placeholder="Filter by path">
<span class="input-group-btn">
<button wicket:id="submit" title="Filter" class="btn btn-default" type="submit"><i class="fa fa-search"></i></button>
</span>
diff --git a/server-core/src/test/java/io/onedev/server/git/command/LogCommandTest.java b/server-core/src/test/java/io/onedev/server/git/command/LogCommandTest.java
index 022de7a836..993659af90 100644
--- a/server-core/src/test/java/io/onedev/server/git/command/LogCommandTest.java
+++ b/server-core/src/test/java/io/onedev/server/git/command/LogCommandTest.java
@@ -3,6 +3,7 @@ package io.onedev.server.git.command;
import static org.junit.Assert.assertEquals;
import java.util.ArrayList;
+import java.util.EnumSet;
import java.util.List;
import org.junit.Test;
@@ -37,7 +38,7 @@ public class LogCommandTest extends AbstractGitTest {
+ "7th line\n"
+ "8th line\n"
+ "9th line\n");
- commit("初始commit");
+ commit("初始commit\n\nfirst line\nsecond line");
rm("测试文件");
addFile("改名后 测试文件", ""
+ "1st line\n"
@@ -67,10 +68,12 @@ public class LogCommandTest extends AbstractGitTest {
+ "7th line\n"
+ "8th line\n"
+ "9th line\n");
- commit("第二个commit");
+ commit("第二个commit\n\nfirst line\n\nsecond line\nthird line\n");
List<GitCommit> commits = new ArrayList<>();
-
+
+ EnumSet<LogCommand.Field> fields = EnumSet.allOf(LogCommand.Field.class);
+ fields.remove(LogCommand.Field.LINE_CHANGES);
new LogCommand(git.getRepository().getDirectory()) {
@Override
@@ -78,24 +81,47 @@ public class LogCommandTest extends AbstractGitTest {
commits.add(commit);
}
- }.revisions(Lists.newArrayList("master")).call();
+ }.revisions(Lists.newArrayList("master")).fields(fields).call();
assertEquals(2, commits.size());
+ GitCommit commit1 = commits.get(1);
+ assertEquals("初始commit", commit1.getSubject());
+ assertEquals("first line\nsecond line", commit1.getBody());
+
+ GitCommit commit2 = commits.get(0);
+ assertEquals(3, commit2.getFileChanges().size());
+ assertEquals("另一个 测试文件", commit2.getFileChanges().get(0).getNewPath());
+ assertEquals("改名后 测试文件", commit2.getFileChanges().get(1).getNewPath());
+ assertEquals("测试文件", commit2.getFileChanges().get(1).getOldPath());
+ assertEquals("第三个 测试文件", commit2.getFileChanges().get(2).getNewPath());
+ assertEquals("第二个commit", commit2.getSubject());
+ assertEquals("first line\n\nsecond line\nthird line", commit2.getBody());
+
+ commits.clear();
+ fields = EnumSet.allOf(LogCommand.Field.class);
+ new LogCommand(git.getRepository().getDirectory()) {
+
+ @Override
+ protected void consume(GitCommit commit) {
+ commits.add(commit);
+ }
+
+ }.revisions(Lists.newArrayList("master")).fields(fields).call();
+
GitCommit commit = commits.get(0);
assertEquals(3, commit.getFileChanges().size());
-
- assertEquals("另一个 测试文件", commit.getFileChanges().get(0).getPath());
+ assertEquals("另一个 测试文件", commit.getFileChanges().get(0).getNewPath());
assertEquals(0, commit.getFileChanges().get(0).getAdditions());
assertEquals(1, commit.getFileChanges().get(0).getDeletions());
- assertEquals("改名后 测试文件", commit.getFileChanges().get(1).getPath());
+ assertEquals("改名后 测试文件", commit.getFileChanges().get(1).getNewPath());
assertEquals("测试文件", commit.getFileChanges().get(1).getOldPath());
- assertEquals("第三个 测试文件", commit.getFileChanges().get(2).getPath());
+ assertEquals("第三个 测试文件", commit.getFileChanges().get(2).getNewPath());
assertEquals(8, commit.getFileChanges().get(2).getAdditions());
- assertEquals(0, commit.getFileChanges().get(2).getDeletions());
+ assertEquals(0, commit.getFileChanges().get(2).getDeletions());
}
}
\ No newline at end of file