The Unified and Autonomous Development Platform
Refactor commit info collecting to make it more performant and accuracy
Robin Shenafcea0748294
diff --git a/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultCodeCommentManager.java b/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultCodeCommentManager.javaindex 919ffcd664..c816a05967 100644--- a/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultCodeCommentManager.java+++ b/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultCodeCommentManager.java@@ -35,6 +35,7 @@ import org.hibernate.query.Query;import com.google.common.base.Preconditions;import com.google.common.collect.Lists;+import com.google.common.collect.Sets;import io.onedev.commons.launcher.loader.Listen;import io.onedev.commons.launcher.loader.ListenerRegistry;@@ -149,11 +150,13 @@ public class DefaultCodeCommentManager extends AbstractEntityManager<CodeCommentMap<CodeComment, PlanarRange> comments = new HashMap<>();Map<String, Map<String, List<CodeComment>>> possibleComments = new HashMap<>();- for (String possibleHistoryPath: commitInfoManager.getHistoryPaths(project, path)) {+ Collection<String> possiblePaths = Sets.newHashSet(path);+ possiblePaths.addAll(commitInfoManager.getHistoryPaths(project, path));+ for (String possiblePath: possiblePaths) {EntityCriteria<CodeComment> criteria = EntityCriteria.of(CodeComment.class);- criteria.add(Restrictions.eq("markPos.path", possibleHistoryPath));+ criteria.add(Restrictions.eq("markPos.path", possiblePath));for (CodeComment comment: query(criteria)) {- if (comment.getMarkPos().getCommit().equals(commitId.name()) && possibleHistoryPath.equals(path)) {+ if (comment.getMarkPos().getCommit().equals(commitId.name()) && possiblePath.equals(path)) {comments.put(comment, comment.getMarkPos().getRange());} else {Map<String, List<CodeComment>> commentsOnCommit =@@ -162,10 +165,10 @@ public class DefaultCodeCommentManager extends AbstractEntityManager<CodeCommentcommentsOnCommit = new HashMap<>();possibleComments.put(comment.getMarkPos().getCommit(), commentsOnCommit);}- List<CodeComment> commentsOnPath = commentsOnCommit.get(possibleHistoryPath);+ List<CodeComment> commentsOnPath = commentsOnCommit.get(possiblePath);if (commentsOnPath == null) {commentsOnPath = new ArrayList<>();- commentsOnCommit.put(possibleHistoryPath, commentsOnPath);+ commentsOnCommit.put(possiblePath, commentsOnPath);}commentsOnPath.add(comment);}diff --git a/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultPullRequestManager.java b/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultPullRequestManager.javaindex 9dd9fe2043..3f88debd5f 100644--- a/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultPullRequestManager.java+++ b/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultPullRequestManager.java@@ -84,7 +84,6 @@ import io.onedev.server.event.pullrequest.PullRequestEvent;import io.onedev.server.event.pullrequest.PullRequestMergePreviewCalculated;import io.onedev.server.event.pullrequest.PullRequestOpened;import io.onedev.server.git.GitUtils;-import io.onedev.server.git.command.FileChange;import io.onedev.server.infomanager.CommitInfoManager;import io.onedev.server.model.Build;import io.onedev.server.model.Group;@@ -140,7 +139,7 @@ public class DefaultPullRequestManager extends AbstractEntityManager<PullRequestprivate static final Logger logger = LoggerFactory.getLogger(DefaultPullRequestManager.class);- private static final int MAX_CONTRIBUTION_FILES = 100;+ private static final int MAX_COMMIT_FILES = 100;private static final int UI_PREVIEW_PRIORITY = 10;@@ -842,7 +841,7 @@ public class DefaultPullRequestManager extends AbstractEntityManager<PullRequestcandidateReviewers.add(user);}- sortByContributions(candidateReviewers, update);+ sortByCommitCount(candidateReviewers, update);for (User user: candidateReviewers) {reviewers.add(user);@@ -858,7 +857,7 @@ public class DefaultPullRequestManager extends AbstractEntityManager<PullRequestif (review == null)candidateReviewers.add(user);}- sortByContributions(candidateReviewers, update);+ sortByCommitCount(candidateReviewers, update);for (User user: candidateReviewers) {reviewers.add(user);PullRequestReview review = new PullRequestReview();@@ -894,34 +893,28 @@ public class DefaultPullRequestManager extends AbstractEntityManager<PullRequest}}- private void sortByContributions(List<User> users, PullRequestUpdate update) {+ private void sortByCommitCount(List<User> users, PullRequestUpdate update) {if (users.size() <= 1)return;CommitInfoManager commitInfoManager = OneDev.getInstance(CommitInfoManager.class);- Map<User, Long> contributions = new HashMap<>();+ Map<User, Long> commitCounts = new HashMap<>();for (User user: users)- contributions.put(user, 0L);+ commitCounts.put(user, 0L);int count = 0;- for (FileChange change: update.getFileChanges()) {- String path = change.getPath();- int edits = change.getAdditions() + change.getDeletions();- if (edits < 0)- edits = 100;- else if (edits == 0)- edits = 1;- int addedContributions = addContributions(contributions, commitInfoManager, path, edits, update);- while (addedContributions == 0) {+ for (String path: update.getChangedFiles()) {+ int addedCommitCount = addCommitCounts(commitCounts, commitInfoManager, path, update);+ while (addedCommitCount == 0) {if (path.contains("/")) {path = StringUtils.substringBeforeLast(path, "/");- addedContributions = addContributions(contributions, commitInfoManager, path, edits, update);+ addedCommitCount = addCommitCounts(commitCounts, commitInfoManager, path, update);} else {- addContributions(contributions, commitInfoManager, "", edits, update);+ addCommitCounts(commitCounts, commitInfoManager, "", update);break;}}- if (++count >= MAX_CONTRIBUTION_FILES)+ if (++count >= MAX_COMMIT_FILES)break;}@@ -929,7 +922,7 @@ public class DefaultPullRequestManager extends AbstractEntityManager<PullRequest@Overridepublic int compare(User o1, User o2) {- if (contributions.get(o1) < contributions.get(o2))+ if (commitCounts.get(o1) < commitCounts.get(o2))return 1;elsereturn -1;@@ -938,17 +931,16 @@ public class DefaultPullRequestManager extends AbstractEntityManager<PullRequest});}- private int addContributions(Map<User, Long> contributions, CommitInfoManager commitInfoManager,- String path, int edits, PullRequestUpdate update) {- int addedContributions = 0;- for (Map.Entry<User, Long> entry: contributions.entrySet()) {+ private int addCommitCounts(Map<User, Long> commitCounts, CommitInfoManager commitInfoManager,+ String path, PullRequestUpdate update) {+ int addedCommitCount = 0;+ for (Map.Entry<User, Long> entry: commitCounts.entrySet()) {User user = entry.getKey();- int pathEdits = commitInfoManager.getEdits(- update.getRequest().getTargetProject(), user, path);- entry.setValue(entry.getValue() + edits*pathEdits);- addedContributions += pathEdits;+ int commitCount = commitInfoManager.getCommitCount(update.getRequest().getTargetProject(), user, path);+ entry.setValue(entry.getValue() + commitCount);+ addedCommitCount += commitCount;}- return addedContributions;+ return addedCommitCount;}private Predicate[] getPredicates(@Nullable Project targetProject,diff --git a/server-core/src/main/java/io/onedev/server/git/GitPostReceiveCallback.java b/server-core/src/main/java/io/onedev/server/git/GitPostReceiveCallback.javaindex 07cf4d53cf..bebf83da92 100644--- a/server-core/src/main/java/io/onedev/server/git/GitPostReceiveCallback.java+++ b/server-core/src/main/java/io/onedev/server/git/GitPostReceiveCallback.java@@ -114,7 +114,7 @@ public class GitPostReceiveCallback extends HttpServlet {RefUpdate refUpdate = GitUtils.getRefUpdate(project.getRepository(), "HEAD");GitUtils.linkRef(refUpdate, refName);}-+listenerRegistry.post(new RefUpdated(project, refName, oldObjectId, newObjectId));field = field.substring(40);diff --git a/server-core/src/main/java/io/onedev/server/git/command/FileChange.java b/server-core/src/main/java/io/onedev/server/git/command/FileChange.javaindex 63a6151488..a39f1a61f4 100644--- a/server-core/src/main/java/io/onedev/server/git/command/FileChange.java+++ b/server-core/src/main/java/io/onedev/server/git/command/FileChange.java@@ -1,31 +1,38 @@package io.onedev.server.git.command;import java.io.Serializable;+import java.util.Collection;+import java.util.HashSet;+import java.util.Set;++import javax.annotation.Nullable;@SuppressWarnings("serial")public class FileChange implements Serializable {private final String oldPath;- private final String path;+ private final String newPath;private final int additions;private final int deletions;- public FileChange(String oldPath, String path, int additions, int deletions) {+ public FileChange(@Nullable String oldPath, @Nullable String newPath, int additions, int deletions) {this.oldPath = oldPath;- this.path = path;+ this.newPath = newPath;this.additions = additions;this.deletions = deletions;}+ @Nullablepublic String getOldPath() {return oldPath;}- public String getPath() {- return path;+ @Nullable+ public String getNewPath() {+ return newPath;}public int getAdditions() {@@ -36,12 +43,13 @@ public class FileChange implements Serializable {return deletions;}- @Override- public String toString() {- if (oldPath != null)- return oldPath + "=>" + path;- else- return path;+ public Collection<String> getPaths() {+ Set<String> paths = new HashSet<>();+ if (getOldPath() != null)+ paths.add(getOldPath());+ if (getNewPath() != null)+ paths.add(getNewPath());+ return paths;}}\ No newline at end of filediff --git a/server-core/src/main/java/io/onedev/server/git/command/GitCommand.java b/server-core/src/main/java/io/onedev/server/git/command/GitCommand.javaindex ff47c3e5f8..4114b33d38 100644--- a/server-core/src/main/java/io/onedev/server/git/command/GitCommand.java+++ b/server-core/src/main/java/io/onedev/server/git/command/GitCommand.java@@ -2,10 +2,12 @@ package io.onedev.server.git.command;import java.io.File;import java.util.Map;+import java.util.StringTokenizer;import javax.annotation.Nullable;import org.apache.commons.lang3.exception.ExceptionUtils;+import org.eclipse.jgit.util.QuotedString;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -97,6 +99,40 @@ public abstract class GitCommand<V> {return cmd;}+ public static FileChange parseNumStats(String line) {+ FileChange change;+ StringTokenizer tokenizer = new StringTokenizer(line, "\t");+ String additionsToken = tokenizer.nextToken();+ int additions = additionsToken.equals("-")?-1:Integer.parseInt(additionsToken);+ String deletionsToken = tokenizer.nextToken();+ int deletions = deletionsToken.equals("-")?-1:Integer.parseInt(deletionsToken);++ String path = tokenizer.nextToken();+ int renameSignIndex = path.indexOf(" => ");+ if (renameSignIndex != -1) {+ int leftBraceIndex = path.indexOf("{");+ int rightBraceIndex = path.indexOf("}");+ if (leftBraceIndex != -1 && rightBraceIndex != -1 && leftBraceIndex<renameSignIndex+ && rightBraceIndex>renameSignIndex) {+ String leftCommon = path.substring(0, leftBraceIndex);+ String rightCommon = path.substring(rightBraceIndex+1);+ String oldPath = leftCommon + path.substring(leftBraceIndex+1, renameSignIndex)+ + rightCommon;+ String newPath = leftCommon + path.substring(renameSignIndex+4, rightBraceIndex)+ + rightCommon;+ change = new FileChange(oldPath, newPath, additions, deletions);+ } else {+ String oldPath = QuotedString.GIT_PATH.dequote(path.substring(0, renameSignIndex));+ String newPath = QuotedString.GIT_PATH.dequote(path.substring(renameSignIndex+4));+ change = new FileChange(oldPath, newPath, additions, deletions);+ }+ } else {+ path = QuotedString.GIT_PATH.dequote(path);+ change = new FileChange(null, path, additions, deletions);+ }+ return change;+ }+protected String getGitExe() {return AppLoader.getInstance(GitConfig.class).getExecutable();}diff --git a/server-core/src/main/java/io/onedev/server/git/command/GitCommit.java b/server-core/src/main/java/io/onedev/server/git/command/GitCommit.javaindex e91e9876e0..0ba4adbb18 100644--- a/server-core/src/main/java/io/onedev/server/git/command/GitCommit.java+++ b/server-core/src/main/java/io/onedev/server/git/command/GitCommit.java@@ -11,8 +11,6 @@ import javax.annotation.Nullable;import org.apache.commons.lang3.StringUtils;import org.eclipse.jgit.lib.PersonIdent;-import com.google.common.base.Preconditions;-import io.onedev.server.git.GitUtils;public class GitCommit implements Serializable {@@ -21,6 +19,10 @@ public class GitCommit implements Serializable {private final String hash;+ private final String subject;++ private final String body;+private final Date commitDate;private final PersonIdent committer;@@ -31,30 +33,44 @@ public class GitCommit implements Serializable {private final List<FileChange> fileChanges;- public GitCommit(String hash, @Nullable Date commitDate, @Nullable PersonIdent committer,- @Nullable PersonIdent author, List<String> parentHashes, @Nullable List<FileChange> fileChanges) {- this.hash = Preconditions.checkNotNull(hash);- this.commitDate = commitDate;+ public GitCommit(String hash, @Nullable List<String> parentHashes, @Nullable PersonIdent committer,+ @Nullable PersonIdent author, @Nullable Date commitDate, @Nullable String subject,+ @Nullable String body, @Nullable List<FileChange> fileChanges) {+ this.hash = hash;+ this.parentHashes = parentHashes;this.committer = committer;this.author = author;- this.parentHashes = Preconditions.checkNotNull(parentHashes);+ this.commitDate = commitDate;+ this.subject = subject;+ this.body = body;this.fileChanges = fileChanges;}-++ @Nullable+ public String getSubject() {+ return subject;+ }++ @Nullable+ public String getBody() {+ return body;+ }++ @Nullablepublic List<String> getParentHashes() {return parentHashes;}- @Nullable+ @Nullablepublic List<FileChange> getFileChanges() {return fileChanges;}- public Collection<String> getChangedFiles() {+ public Collection<String> getChangedFiles() {Collection<String> changedFiles = new HashSet<>();for (FileChange change: getFileChanges()) {- if (change.getPath() != null)- changedFiles.add(change.getPath());+ if (change.getNewPath() != null)+ changedFiles.add(change.getNewPath());if (change.getOldPath() != null)changedFiles.add(change.getOldPath());}@@ -65,7 +81,7 @@ public class GitCommit implements Serializable {return hash;}- @Nullable+ @Nullablepublic PersonIdent getCommitter() {return committer;}@@ -102,6 +118,10 @@ public class GitCommit implements Serializable {public String hash;+ public String subject;++ public String body;+public String authorName;public String authorEmail;@@ -114,6 +134,8 @@ public class GitCommit implements Serializable {public Date committerDate;+ public Date commitDate;+public List<String> parentHashes;public List<FileChange> fileChanges;@@ -130,8 +152,15 @@ public class GitCommit implements Serializable {author = GitUtils.newPersonIdent(authorName, authorEmail, authorDate);elseauthor = null;++ if (subject != null)+ subject = subject.trim();++ if (body != null)+ body = body.trim();- return new GitCommit(hash, committerDate, committer, author, parentHashes, fileChanges);+ return new GitCommit(hash, parentHashes, committer, author, commitDate,+ subject, body, fileChanges);}}diff --git a/server-core/src/main/java/io/onedev/server/git/command/ListFileChangesCommand.java b/server-core/src/main/java/io/onedev/server/git/command/ListFileChangesCommand.javadeleted file mode 100644index 7f66c723a2..0000000000--- a/server-core/src/main/java/io/onedev/server/git/command/ListFileChangesCommand.java+++ /dev/null@@ -1,99 +0,0 @@-package io.onedev.server.git.command;--import java.io.File;-import java.util.ArrayList;-import java.util.Collection;-import java.util.StringTokenizer;--import org.eclipse.jgit.util.QuotedString;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--import com.google.common.base.Preconditions;--import io.onedev.commons.utils.command.Commandline;-import io.onedev.commons.utils.command.LineConsumer;--public class ListFileChangesCommand extends GitCommand<Collection<FileChange>> {-- private static final Logger logger = LoggerFactory.getLogger(ListFileChangesCommand.class);-- private String fromRev;-- private String toRev;-- private String path;-- public ListFileChangesCommand(File gitDir) {- super(gitDir);- }-- public ListFileChangesCommand fromRev(String fromRev) {- this.fromRev = fromRev;- return this;- }-- public ListFileChangesCommand toRev(String toRev) {- this.toRev = toRev;- return this;- }-- public ListFileChangesCommand path(String path) {- this.path = path;- return this;- }-- @Override- public Collection<FileChange> call() {- Preconditions.checkNotNull(toRev, "toRev has to be specified.");- Preconditions.checkNotNull(fromRev, "fromRev has to be specified.");-- final Collection<FileChange> fileChanges = new ArrayList<>();-- Commandline cmd = cmd();-- cmd.addArgs("diff", "--numstat", "--no-renames", fromRev + ".." + toRev);-- if (path != null)- cmd.addArgs("--", path);-- cmd.execute(new LineConsumer() {-- @Override- public void consume(String line) {- if (line.contains("\t")) {- StringTokenizer tokenizer = new StringTokenizer(line, "\t");-- int additions;- String field = tokenizer.nextToken();- if (field.equals("-"))- additions = -1;- else- additions = Integer.parseInt(field);-- int deletions;- field = tokenizer.nextToken();- if (field.equals("-"))- deletions = -1;- else- deletions = Integer.parseInt(field);-- String file = QuotedString.GIT_PATH.dequote(tokenizer.nextToken());-- fileChanges.add(new FileChange(null, file, additions, deletions));- }- }-- }, new LineConsumer() {-- @Override- public void consume(String line) {- logger.error(line);- }-- }).checkReturnCode();-- return fileChanges;- }--}diff --git a/server-core/src/main/java/io/onedev/server/git/command/ListNumStatsCommand.java b/server-core/src/main/java/io/onedev/server/git/command/ListNumStatsCommand.javanew file mode 100644index 0000000000..23ae91fa7d--- /dev/null+++ b/server-core/src/main/java/io/onedev/server/git/command/ListNumStatsCommand.java@@ -0,0 +1,68 @@+package io.onedev.server.git.command;++import java.io.File;+import java.util.ArrayList;+import java.util.List;++import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++import com.google.common.base.Preconditions;++import io.onedev.commons.utils.command.Commandline;+import io.onedev.commons.utils.command.LineConsumer;++public class ListNumStatsCommand extends GitCommand<List<FileChange>> {++ private static final Logger logger = LoggerFactory.getLogger(ListNumStatsCommand.class);++ private String fromRev;++ private String toRev;++ public ListNumStatsCommand(File gitDir) {+ super(gitDir);+ }++ public ListNumStatsCommand fromRev(String fromRev) {+ this.fromRev = fromRev;+ return this;+ }++ public ListNumStatsCommand toRev(String toRev) {+ this.toRev = toRev;+ return this;+ }++ @Override+ public List<FileChange> call() {+ Preconditions.checkNotNull(toRev, "toRev has to be specified.");+ Preconditions.checkNotNull(fromRev, "fromRev has to be specified.");++ List<FileChange> fileChanges = new ArrayList<>();++ Commandline cmd = cmd();++ cmd.addArgs("-c", "diff.renameLimit=1000", "diff", "--numstat",+ "--find-renames", fromRev + ".." + toRev);++ cmd.execute(new LineConsumer() {++ @Override+ public void consume(String line) {+ fileChanges.add(parseNumStats(line));+ }++ }, new LineConsumer() {++ @Override+ public void consume(String line) {+ logger.error(line);+ }++ }).checkReturnCode();++ return fileChanges;+ }++}\ No newline at end of filediff --git a/server-core/src/main/java/io/onedev/server/git/command/LogCommand.java b/server-core/src/main/java/io/onedev/server/git/command/LogCommand.javaindex 77bd94b87e..5b720b0061 100644--- a/server-core/src/main/java/io/onedev/server/git/command/LogCommand.java+++ b/server-core/src/main/java/io/onedev/server/git/command/LogCommand.java@@ -5,6 +5,7 @@ import java.util.ArrayList;import java.util.EnumSet;import java.util.List;import java.util.StringTokenizer;+import java.util.concurrent.atomic.AtomicBoolean;import java.util.concurrent.atomic.AtomicReference;import org.eclipse.jgit.util.QuotedString;@@ -22,9 +23,13 @@ public abstract class LogCommand extends GitCommand<Void> {private static final Logger logger = LoggerFactory.getLogger(LogCommand.class);- public enum Field {PARENTS, AUTHOR, COMMITTER, FILE_CHANGES};+ public enum Field {PARENTS, AUTHOR, COMMITTER, COMMIT_DATE, SUBJECT, BODY, FILE_CHANGES, LINE_CHANGES};- private EnumSet<Field> fields = EnumSet.allOf(Field.class);+ private static final String BODY_END = "$<#BodyEnd#>$";++ private boolean firstParent;++ private EnumSet<Field> fields = EnumSet.noneOf(Field.class);private List<String> revisions = new ArrayList<>();@@ -41,15 +46,16 @@ public abstract class LogCommand extends GitCommand<Void> {return this;}- public EnumSet<Field> fields() {- return fields;- }-public LogCommand fields(EnumSet<Field> fields) {this.fields = fields;return this;}-++ public LogCommand firstParent(boolean firstParent) {+ this.firstParent = firstParent;+ return this;+ }+@Overridepublic Void call() {Preconditions.checkArgument(!revisions.isEmpty(), "Log revisions have to be specified");@@ -73,31 +79,65 @@ public abstract class LogCommand extends GitCommand<Void> {+ "committerEmail:%cE %n"+ "committerDate:%cd %n";}-- if (fields.contains(Field.FILE_CHANGES))+ if (fields.contains(Field.COMMIT_DATE))+ format += "commitDate:%cd %n";+ if (fields.contains(Field.SUBJECT))+ format += "subject:%s %n";+ if (fields.contains(Field.BODY))+ format += "body:%b" + BODY_END + "%n";++ if (fields.contains(Field.LINE_CHANGES))cmd.addArgs("-c", "diff.renameLimit=1000", "log", "--numstat", "--find-renames");+ else if (fields.contains(Field.FILE_CHANGES))+ cmd.addArgs("-c", "diff.renameLimit=1000", "log", "--name-status", "--find-renames");elsecmd.addArgs("log");cmd.addArgs("--format=" + format, "--date=raw");-+ if (firstParent) {+ cmd.addArgs("--first-parent");+ if (fields.contains(Field.LINE_CHANGES) || fields.contains(Field.FILE_CHANGES))+ cmd.addArgs("-m");+ }+for (String revision: revisions)cmd.addArgs(revision);AtomicReference<GitCommit.Builder> commitBuilderRef = new AtomicReference<>(null);+ AtomicBoolean inBodyRef = new AtomicBoolean(false);cmd.execute(new LineConsumer() {@Overridepublic void consume(String line) {- if (line.startsWith("hash:")) {+ if (inBodyRef.get()) {+ line = line.trim();+ if (line.endsWith(BODY_END)) {+ commitBuilderRef.get().body += "\n" + line.substring(0, line.length()-BODY_END.length());+ inBodyRef.set(false);+ } else {+ commitBuilderRef.get().body += "\n" + line;+ }+ } else if (line.startsWith("body:")) {+ line = line.substring("body:".length()).trim();+ if (line.endsWith(BODY_END)) {+ commitBuilderRef.get().body = line.substring(0, line.length()-BODY_END.length());+ } else {+ commitBuilderRef.get().body = line;+ inBodyRef.set(true);+ }+ } else if (line.startsWith("hash:")) {if (commitBuilderRef.get() != null)LogCommand.this.consume(commitBuilderRef.get().build());commitBuilderRef.set(new GitCommit.Builder());if (fields.contains(Field.PARENTS))commitBuilderRef.get().parentHashes = new ArrayList<>();- if (fields.contains(Field.FILE_CHANGES))+ if (fields.contains(Field.FILE_CHANGES) || fields.contains(Field.LINE_CHANGES))commitBuilderRef.get().fileChanges = new ArrayList<>();commitBuilderRef.get().hash = line.substring("hash:".length()).trim();+ } else if (line.startsWith("parents:")) {+ Splitter splitter = Splitter.on(" ").omitEmptyStrings().trimResults();+ for (String each: splitter.split(line.substring("parents:".length())))+ commitBuilderRef.get().parentHashes.add(each);} else if (line.startsWith("author:")) {commitBuilderRef.get().authorName = line.substring("author:".length()).trim();} else if (line.startsWith("committer:")) {@@ -106,48 +146,43 @@ public abstract class LogCommand extends GitCommand<Void> {commitBuilderRef.get().authorEmail = line.substring("authorEmail:".length()).trim();} else if (line.startsWith("committerEmail:")) {commitBuilderRef.get().committerEmail = line.substring("committerEmail:".length()).trim();- } else if (line.startsWith("parents:")) {- Splitter splitter = Splitter.on(" ").omitEmptyStrings().trimResults();- for (String each: splitter.split(line.substring("parents:".length())))- commitBuilderRef.get().parentHashes.add(each);} else if (line.startsWith("committerDate:")) {commitBuilderRef.get().committerDate =GitUtils.parseRawDate(line.substring("committerDate:".length()).trim());} else if (line.startsWith("authorDate:")) {commitBuilderRef.get().authorDate =GitUtils.parseRawDate(line.substring("authorDate:".length()).trim());- } else if (line.trim().length() != 0) {- StringTokenizer tokenizer = new StringTokenizer(line, "\t");- String additionsToken = tokenizer.nextToken();- int additions = additionsToken.equals("-")?-1:Integer.parseInt(additionsToken);- String deletionsToken = tokenizer.nextToken();- int deletions = deletionsToken.equals("-")?-1:Integer.parseInt(deletionsToken);-- String path = tokenizer.nextToken();- int renameSignIndex = path.indexOf(" => ");- if (renameSignIndex != -1) {- int leftBraceIndex = path.indexOf("{");- int rightBraceIndex = path.indexOf("}");- if (leftBraceIndex != -1 && rightBraceIndex != -1 && leftBraceIndex<renameSignIndex- && rightBraceIndex>renameSignIndex) {- String leftCommon = path.substring(0, leftBraceIndex);- String rightCommon = path.substring(rightBraceIndex+1);- String oldPath = leftCommon + path.substring(leftBraceIndex+1, renameSignIndex)- + rightCommon;- String newPath = leftCommon + path.substring(renameSignIndex+4, rightBraceIndex)- + rightCommon;- commitBuilderRef.get().fileChanges.add(- new FileChange(oldPath, newPath, additions, deletions));- } else {- String oldPath = QuotedString.GIT_PATH.dequote(path.substring(0, renameSignIndex));- String newPath = QuotedString.GIT_PATH.dequote(path.substring(renameSignIndex+4));- commitBuilderRef.get().fileChanges.add(- new FileChange(oldPath, newPath, additions, deletions));- }+ } else if (line.startsWith("commitDate:")) {+ commitBuilderRef.get().commitDate =+ GitUtils.parseRawDate(line.substring("commitDate:".length()).trim());+ } else if (line.startsWith("subject:")) {+ commitBuilderRef.get().subject = line.substring("subject:".length()).trim();+ } else if (line.trim().length() != 0 && line.contains("\t")) {+ FileChange change;+ if (fields.contains(Field.LINE_CHANGES)) {+ change = parseNumStats(line);} else {- path = QuotedString.GIT_PATH.dequote(path);- commitBuilderRef.get().fileChanges.add(new FileChange(null, path, additions, deletions));+ StringTokenizer tokenizer = new StringTokenizer(line, "\t");+ String statusCode = tokenizer.nextToken();+ if (statusCode.startsWith("R")) {+ String oldPath = QuotedString.GIT_PATH.dequote(tokenizer.nextToken("\t"));+ String newPath = QuotedString.GIT_PATH.dequote(tokenizer.nextToken("\t"));+ change = new FileChange(oldPath, newPath, -1, -1);+ } else if (statusCode.equals("M") || statusCode.equals("T")) {+ String path = QuotedString.GIT_PATH.dequote(tokenizer.nextToken("\t"));+ change = new FileChange(path, path, -1, -1);+ } else if (statusCode.equals("D")) {+ String oldPath = QuotedString.GIT_PATH.dequote(tokenizer.nextToken("\t"));+ change = new FileChange(oldPath, null, -1, -1);+ } else if (statusCode.equals("A")) {+ String newPath = QuotedString.GIT_PATH.dequote(tokenizer.nextToken("\t"));+ change = new FileChange(null, newPath, -1, -1);+ } else {+ change = null;+ }}+ if (change != null)+ commitBuilderRef.get().fileChanges.add(change);}}diff --git a/server-core/src/main/java/io/onedev/server/infomanager/CommitInfoManager.java b/server-core/src/main/java/io/onedev/server/infomanager/CommitInfoManager.javaindex 43c6652f85..531c49772e 100644--- a/server-core/src/main/java/io/onedev/server/infomanager/CommitInfoManager.java+++ b/server-core/src/main/java/io/onedev/server/infomanager/CommitInfoManager.java@@ -17,7 +17,7 @@ public interface CommitInfoManager {List<String> getFiles(Project project);- int getEdits(Project project, User user, String path);+ int getCommitCount(Project project, User user, String path);int getCommitCount(Project project);diff --git a/server-core/src/main/java/io/onedev/server/infomanager/DefaultCommitInfoManager.java b/server-core/src/main/java/io/onedev/server/infomanager/DefaultCommitInfoManager.javaindex 4197a777f1..8a711e0955 100644--- a/server-core/src/main/java/io/onedev/server/infomanager/DefaultCommitInfoManager.java+++ b/server-core/src/main/java/io/onedev/server/infomanager/DefaultCommitInfoManager.java@@ -8,18 +8,18 @@ import java.util.ArrayList;import java.util.Collection;import java.util.Collections;import java.util.Comparator;+import java.util.EnumSet;import java.util.HashMap;import java.util.HashSet;import java.util.Iterator;import java.util.List;import java.util.Map;-import java.util.Optional;import java.util.Set;import java.util.Stack;import java.util.concurrent.ConcurrentHashMap;-import java.util.concurrent.ExecutorService;-import java.util.concurrent.SynchronousQueue;+import java.util.concurrent.atomic.AtomicInteger;import java.util.concurrent.atomic.AtomicReference;+import java.util.function.Consumer;import javax.inject.Inject;import javax.inject.Singleton;@@ -37,12 +37,13 @@ import org.slf4j.Logger;import org.slf4j.LoggerFactory;import com.google.common.base.Charsets;+import com.google.common.base.Preconditions;import com.google.common.base.Splitter;import io.onedev.commons.launcher.loader.Listen;import io.onedev.commons.launcher.loader.ListenerRegistry;-import io.onedev.commons.utils.ExceptionUtils;import io.onedev.commons.utils.FileUtils;+import io.onedev.commons.utils.Pair;import io.onedev.commons.utils.PathUtils;import io.onedev.commons.utils.StringUtils;import io.onedev.commons.utils.concurrent.Prioritized;@@ -58,6 +59,7 @@ import io.onedev.server.git.GitUtils;import io.onedev.server.git.NameAndEmail;import io.onedev.server.git.command.FileChange;import io.onedev.server.git.command.GitCommit;+import io.onedev.server.git.command.ListNumStatsCommand;import io.onedev.server.git.command.LogCommand;import io.onedev.server.git.command.RevListCommand;import io.onedev.server.git.command.RevListCommand.Order;@@ -68,6 +70,7 @@ import io.onedev.server.persistence.SessionManager;import io.onedev.server.persistence.annotation.Sessional;import io.onedev.server.storage.StorageManager;import io.onedev.server.util.Day;+import io.onedev.server.util.ElementPumper;import io.onedev.server.util.IssueUtils;import io.onedev.server.util.work.BatchWorkManager;import io.onedev.server.util.work.BatchWorker;@@ -87,7 +90,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager implemeprivate static final Logger logger = LoggerFactory.getLogger(DefaultCommitInfoManager.class);- private static final int INFO_VERSION = 7;+ private static final int INFO_VERSION = 8;private static final long LOG_FILE_SIZE = 256*1024;@@ -105,7 +108,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager implemeprivate static final String FIX_COMMITS_STORE = "fixCommits";- private static final String EDITS_STORE = "edits";+ private static final String COMMIT_COUNTS_STORE = "commitCounts";private static final String HISTORY_PATHS_STORE = "historyPaths";@@ -279,27 +282,24 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager implemeprivate final SessionManager sessionManager;- private final ExecutorService executorService;-private final ListenerRegistry listenerRegistry;private final IssueManager issueManager;private final Map<Long, List<String>> filesCache = new ConcurrentHashMap<>();- private final Map<Long, Integer> commitCountCache = new ConcurrentHashMap<>();+ private final Map<Long, Integer> totalCommitCountCache = new ConcurrentHashMap<>();private final Map<Long, List<NameAndEmail>> usersCache = new ConcurrentHashMap<>();@Injectpublic DefaultCommitInfoManager(ProjectManager projectManager, StorageManager storageManager,- BatchWorkManager batchWorkManager, SessionManager sessionManager, ExecutorService executorService,+ BatchWorkManager batchWorkManager, SessionManager sessionManager,ListenerRegistry listenerRegistry, IssueManager issueManager) {this.projectManager = projectManager;this.storageManager = storageManager;this.batchWorkManager = batchWorkManager;this.sessionManager = sessionManager;- this.executorService = executorService;this.listenerRegistry = listenerRegistry;this.issueManager = issueManager;}@@ -318,285 +318,208 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager implemeEnvironment env = getEnv(project.getId().toString());Store defaultStore = getStore(env, DEFAULT_STORE);Store commitsStore = getStore(env, COMMITS_STORE);- Store editsStore = getStore(env, EDITS_STORE);+ Store commitCountsStore = getStore(env, COMMIT_COUNTS_STORE);Store historyPathsStore = getStore(env, HISTORY_PATHS_STORE);Store pathToIndexStore = getStore(env, PATH_TO_INDEX_STORE);Store indexToPathStore = getStore(env, INDEX_TO_PATH_STORE);Store emailToIndexStore = getStore(env, EMAIL_TO_INDEX_STORE);Store indexToUserStore = getStore(env, INDEX_TO_USER_STORE);- Store dailyContributionsStore = getStore(env, DAILY_CONTRIBUTIONS_STORE);Store fixCommitsStore = getStore(env, FIX_COMMITS_STORE);+ Store dailyContributionsStore = getStore(env, DAILY_CONTRIBUTIONS_STORE);Repository repository = project.getRepository();Collection<Long> fixedIssueNumbers = new HashSet<>();- env.executeInTransaction(new TransactionalExecutable() {+ Pair<byte[], ObjectId> result = env.computeInTransaction(new TransactionalComputable<Pair<byte[], ObjectId>>() {@Override- public void execute(Transaction txn) {+ public Pair<byte[], ObjectId> compute(Transaction txn) {ByteIterable commitKey = new CommitByteIterable(commitId);byte[] commitBytes = readBytes(commitsStore, txn, commitKey);- if (!isCommitCollected(commitBytes)) {- int commitCount = readInt(defaultStore, txn, COMMIT_COUNT_KEY, 0);+ ObjectId lastCommitId;+ byte[] lastCommitBytes = readBytes(defaultStore, txn, LAST_COMMIT_KEY);+ if (lastCommitBytes != null) {+ lastCommitId = ObjectId.fromRaw(lastCommitBytes);+ if (!repository.hasObject(lastCommitId))+ lastCommitId = null;+ } else {+ lastCommitId = null;+ }++ return new Pair<>(commitBytes, lastCommitId);+ }+ });++ if (!isCommitCollected(result.getFirst())) {+ AtomicReference<ObjectId> lastCommitIdRef = new AtomicReference<>(result.getSecond());+ RevListCommand revList = new RevListCommand(project.getGitDir());+ List<String> revisions = new ArrayList<>();+ revisions.add(commitId.name());+ if (lastCommitIdRef.get() != null)+ revisions.add("^" + lastCommitIdRef.get().name());+ revList.revisions(revisions).order(Order.TOPO);++ List<ObjectId> historyIds = new ArrayList<>();+ for (String commitHash: revList.call())+ historyIds.add(ObjectId.fromString(commitHash));++ revList = new RevListCommand(project.getGitDir());+ revList.revisions(revisions).order(null).firstParent(true);++ Set<ObjectId> firstParentIds = new HashSet<>();+ for (String commitHash: revList.call())+ firstParentIds.add(ObjectId.fromString(commitHash));++ /*+ * Instead of collecting information of master branch all at once, we identify some+ * intermediate commits and collect the information using these intermediate commits+ * multiple times for two reasons:+ * 1. Use less memory+ * 2. Commit Exodus transaction sooner so user can use auto-completion when search+ * commits even if collection is not done yet+ */+ List<ObjectId> intermediateCommitIds = new ArrayList<>();+ int count = 0;+ for (ObjectId historyId: historyIds) {+ count++;+ /*+ * Only use intermediate commits that are part of first parent chain. This+ * makes sure that subsequent intermediate commits are always ancestor of+ * current intermediate commit (after reverse done below), to avoid+ * collecting some commits multiple times+ */+ if (count > COLLECT_BATCH_SIZE && firstParentIds.contains(historyId)) {+ intermediateCommitIds.add(historyId);+ count = 0;+ }+ }++ Collections.reverse(intermediateCommitIds);+ intermediateCommitIds.add(commitId);++ for(ObjectId intermediateCommitId: intermediateCommitIds) {+ env.executeInTransaction(new TransactionalExecutable() {- try (RevWalk revWalk = new RevWalk(project.getRepository())) {- RevCommit commit = revWalk.lookupCommit(commitId);- revWalk.markStart(commit);+ @SuppressWarnings("unchecked")+ @Override+ public void execute(Transaction txn) {+ AtomicInteger totalCommitCount = new AtomicInteger(readInt(defaultStore, txn, COMMIT_COUNT_KEY, 0));- byte[] lastCommitBytes = readBytes(defaultStore, txn, LAST_COMMIT_KEY);- if (lastCommitBytes != null) {- ObjectId lastCommitId = ObjectId.fromRaw(lastCommitBytes);- if (repository.hasObject(lastCommitId))- revWalk.markUninteresting(revWalk.lookupCommit(lastCommitId));- }+ NextIndex nextIndex = new NextIndex();+ nextIndex.user = readInt(defaultStore, txn, NEXT_USER_INDEX_KEY, 0);+ nextIndex.path = readInt(defaultStore, txn, NEXT_PATH_INDEX_KEY, 0);- RevCommit nextCommit = revWalk.next();- while (nextCommit != null) {- ByteIterable nextCommitKey = new CommitByteIterable(nextCommit);- byte[] nextCommitBytes = readBytes(commitsStore, txn, nextCommitKey);-- if (!isCommitCollected(nextCommitBytes)) {- byte[] newNextCommitBytes;- if (nextCommitBytes == null) {- newNextCommitBytes = new byte[1];- } else {- newNextCommitBytes = new byte[1+nextCommitBytes.length];- System.arraycopy(nextCommitBytes, 0, newNextCommitBytes, 1, nextCommitBytes.length);- }-- commitsStore.put(txn, nextCommitKey, new ArrayByteIterable(newNextCommitBytes));-- commitCount++;-- for (RevCommit parentCommit: nextCommit.getParents()) {- ByteIterable parentCommitKey = new CommitByteIterable(parentCommit);- byte[] parentCommitBytes = readBytes(commitsStore, txn, parentCommitKey);- byte[] newParentCommitBytes;- if (parentCommitBytes != null) {- newParentCommitBytes = new byte[parentCommitBytes.length+20];- System.arraycopy(parentCommitBytes, 0, newParentCommitBytes, 0, parentCommitBytes.length);- } else {- newParentCommitBytes = new byte[20];- }- nextCommit.copyRawTo(newParentCommitBytes, newParentCommitBytes.length-20);- commitsStore.put(txn, parentCommitKey, new ArrayByteIterable(newParentCommitBytes));- }-- for (Long issueNumber: IssueUtils.parseFixedIssueNumbers(nextCommit.getFullMessage())) {- ByteIterable issueKey = new LongByteIterable(issueNumber);- Collection<ObjectId> fixCommits = readCommits(fixCommitsStore, txn, issueKey);-- boolean addNextCommit = true;- for (Iterator<ObjectId> it = fixCommits.iterator(); it.hasNext();) {- ObjectId fixCommit = it.next();- if (GitUtils.isMergedInto(project.getRepository(), null, fixCommit, nextCommit)) {- it.remove();- } else if (GitUtils.isMergedInto(project.getRepository(), null, nextCommit, fixCommit)) {- addNextCommit = false;- break;- }- }- if (addNextCommit) {- fixCommits.add(nextCommit);- fixedIssueNumbers.add(issueNumber);- }- writeCommits(fixCommitsStore, txn, issueKey, fixCommits);- }- }- nextCommit = revWalk.next();- }- writeInt(defaultStore, txn, COMMIT_COUNT_KEY, commitCount);- commitCountCache.remove(project.getId());+ Map<Long, Integer> commitCountCache = new HashMap<>();- defaultStore.put(txn, LAST_COMMIT_KEY, new CommitByteIterable(commitId));- } catch (IOException e) {- throw new RuntimeException(e);- }- }- }-- });-- for (Long issueNumber: fixedIssueNumbers) {- Issue issue = issueManager.find(project, issueNumber);- if (issue != null)- listenerRegistry.post(new IssueCommitted(issue));- }+ Set<NameAndEmail> users;+ byte[] userBytes = readBytes(defaultStore, txn, USERS_KEY);+ if (userBytes != null)+ users = (Set<NameAndEmail>) SerializationUtils.deserialize(userBytes);+ else+ users = new HashSet<>();- if (GitUtils.branch2ref(project.getDefaultBranch()).equals(refName)) {- ObjectId lastCommitId =- env.computeInReadonlyTransaction(new TransactionalComputable<ObjectId>() {+ Map<String, Long> files;+ byte[] fileBytes = readBytes(defaultStore, txn, FILES_KEY);+ if (fileBytes != null)+ files = (Map<String, Long>) SerializationUtils.deserialize(fileBytes);+ else+ files = new HashMap<>();- @Override- public ObjectId compute(Transaction txn) {- byte[] lastCommitBytes = readBytes(defaultStore, txn, LAST_COMMIT_OF_DEFAULT_BRANCH_KEY);- if (lastCommitBytes != null)- return ObjectId.fromRaw(lastCommitBytes);- else- return null;- }-- });+ new ElementPumper<GitCommit>() {- if (!commitId.equals(lastCommitId)) {- RevListCommand revList = new RevListCommand(project.getGitDir());- List<String> revisions = new ArrayList<>();- revisions.add(commitId.name());- if (lastCommitId != null && repository.hasObject(lastCommitId))- revisions.add("^" + lastCommitId.name());- revList.revisions(revisions).order(Order.TOPO);-- List<ObjectId> historyIds = new ArrayList<>();- for (String commitHash: revList.call())- historyIds.add(ObjectId.fromString(commitHash));-- revList = new RevListCommand(project.getGitDir());- revList.order(null).firstParent(true);-- Set<ObjectId> firstParentIds = new HashSet<>();- for (String commitHash: revList.call())- firstParentIds.add(ObjectId.fromString(commitHash));+ @Override+ public void generate(Consumer<GitCommit> consumer) {+ List<String> revisions = new ArrayList<>();+ revisions.add(intermediateCommitId.name());- /*- * Instead of collecting information of master branch all at once, we identify some- * intermediate commits and collect the information using these intermediate commits- * multiple times for two reasons:- * 1. Use less memory- * 2. Commit Exodus transaction sooner so user can use auto-completion when search- * commits even if collection is not done yet- */- List<ObjectId> intermediateIds = new ArrayList<>();- int count = 0;- for (ObjectId historyId: historyIds) {- count++;- /*- * Only use intermediate commits that are part of first parent chain. This- * makes sure that subsequent intermediate commits are always ancestor of- * current intermediate commit (after reverse done below), to avoid- * collecting some commits multiple times- */- if (count > COLLECT_BATCH_SIZE && firstParentIds.contains(historyId)) {- intermediateIds.add(historyId);- count = 0;- }- }+ if (lastCommitIdRef.get() != null)+ revisions.add("^" + lastCommitIdRef.get().name());- Collections.reverse(intermediateIds);- intermediateIds.add(commitId);-- historyIds = null;- firstParentIds = null;-- for(ObjectId currentCommitId: intermediateIds) {- env.executeInTransaction(new TransactionalExecutable() {-- @SuppressWarnings("unchecked")- @Override- public void execute(Transaction txn) {- NextIndex nextIndex = new NextIndex();- nextIndex.user = readInt(defaultStore, txn, NEXT_USER_INDEX_KEY, 0);- nextIndex.path = readInt(defaultStore, txn, NEXT_PATH_INDEX_KEY, 0);-- Map<Long, Integer> editsCache = new HashMap<>();-- Set<NameAndEmail> users;- byte[] userBytes = readBytes(defaultStore, txn, USERS_KEY);- if (userBytes != null)- users = (Set<NameAndEmail>) SerializationUtils.deserialize(userBytes);- else- users = new HashSet<>();-- Map<String, Long> files;- byte[] fileBytes = readBytes(defaultStore, txn, FILES_KEY);- if (fileBytes != null)- files = (Map<String, Long>) SerializationUtils.deserialize(fileBytes);- else- files = new HashMap<>();-- Map<Integer, Map<String, Integer>> lineStats;- byte[] bytesOfLineStats = readBytes(defaultStore, txn, LINE_STATS_KEY);- if (bytesOfLineStats != null) {- lineStats = (Map<Integer, Map<String, Integer>>) SerializationUtils.deserialize(- bytesOfLineStats);- } else {- lineStats = new HashMap<>();- }-- Map<Integer, Map<Integer, Contribution>> dailyContributionsCache = new HashMap<>();- Map<Integer, Contribution> overallContributions =- deserializeContributions(readBytes(defaultStore, txn, OVERALL_CONTRIBUTIONS_KEY));-- /*- * Use a synchronous queue to achieve below purpose:- * 1. Add commit to Xodus transactional store in the same thread opening the transaction- * as this is required by Xodus- * 2. Do not pile up commits to use minimal memory- */- SynchronousQueue<Optional<GitCommit>> queue = new SynchronousQueue<>();- AtomicReference<Exception> logException = new AtomicReference<>(null);-- List<String> revisions = new ArrayList<>();- revisions.add(currentCommitId.name());-- ObjectId lastCommitId;- byte[] lastCommitBytes = readBytes(defaultStore, txn, LAST_COMMIT_OF_DEFAULT_BRANCH_KEY);- if (lastCommitBytes != null)- lastCommitId = ObjectId.fromRaw(lastCommitBytes);- else- lastCommitId = null;- if (lastCommitId != null && repository.hasObject(lastCommitId))- revisions.add("^" + lastCommitId.name());-- LogCommand log = new LogCommand(project.getGitDir()) {-- @Override- protected void consume(GitCommit commit) {- try {- queue.put(Optional.of(commit));- } catch (InterruptedException e) {+ EnumSet<LogCommand.Field> fields = EnumSet.allOf(LogCommand.Field.class);+ fields.remove(LogCommand.Field.LINE_CHANGES);+ new LogCommand(project.getGitDir()) {++ @Override+ protected void consume(GitCommit commit) {+ consumer.accept(commit);}- }++ }.revisions(revisions).fields(fields).call();+ }++ @Override+ public void process(GitCommit currentCommit) {+ ObjectId currentCommitId = ObjectId.fromString(currentCommit.getHash());+ ByteIterable currentCommitKey = new CommitByteIterable(currentCommitId);+ byte[] currentCommitBytes = readBytes(commitsStore, txn, currentCommitKey);- }.revisions(revisions);-- executorService.execute(new Runnable() {-- @Override- public void run() {- try {- log.call();- } catch (Exception e) {- logException.set(e);- } finally {- try {- queue.put(Optional.empty());- } catch (InterruptedException e) {+ if (!isCommitCollected(currentCommitBytes)) {+ totalCommitCount.incrementAndGet();++ byte[] newCurrentCommitBytes;+ if (currentCommitBytes == null) {+ newCurrentCommitBytes = new byte[1];+ } else {+ newCurrentCommitBytes = new byte[1+currentCommitBytes.length];+ System.arraycopy(currentCommitBytes, 0, newCurrentCommitBytes, 1, currentCommitBytes.length);+ }++ commitsStore.put(txn, currentCommitKey, new ArrayByteIterable(newCurrentCommitBytes));++ for (String parentCommitHash: currentCommit.getParentHashes()) {+ ByteIterable parentCommitKey = new CommitByteIterable(ObjectId.fromString(parentCommitHash));+ byte[] parentCommitBytes = readBytes(commitsStore, txn, parentCommitKey);+ byte[] newParentCommitBytes;+ if (parentCommitBytes != null) {+ newParentCommitBytes = new byte[parentCommitBytes.length+20];+ System.arraycopy(parentCommitBytes, 0, newParentCommitBytes, 0, parentCommitBytes.length);+ } else {+ newParentCommitBytes = new byte[20];}+ currentCommitId.copyRawTo(newParentCommitBytes, newParentCommitBytes.length-20);+ commitsStore.put(txn, parentCommitKey, new ArrayByteIterable(newParentCommitBytes));+ }++ String commitMessage = currentCommit.getSubject();+ if (currentCommit.getBody() != null)+ commitMessage += "\n\n" + currentCommit.getBody();++ for (Long issueNumber: IssueUtils.parseFixedIssueNumbers(commitMessage)) {+ ByteIterable issueKey = new LongByteIterable(issueNumber);+ Collection<ObjectId> fixCommits = readCommits(fixCommitsStore, txn, issueKey);++ boolean addNextCommit = true;+ for (Iterator<ObjectId> it = fixCommits.iterator(); it.hasNext();) {+ ObjectId fixCommit = it.next();+ if (GitUtils.isMergedInto(project.getRepository(), null, fixCommit, currentCommitId)) {+ it.remove();+ } else if (GitUtils.isMergedInto(project.getRepository(), null, currentCommitId, fixCommit)) {+ addNextCommit = false;+ break;+ }+ }+ if (addNextCommit) {+ fixCommits.add(currentCommitId);+ fixedIssueNumbers.add(issueNumber);+ }+ writeCommits(fixCommitsStore, txn, issueKey, fixCommits);}- }-- });-- try {- Optional<GitCommit> logCommitOptional = queue.take();- while (logCommitOptional.isPresent()) {- GitCommit logCommit = logCommitOptional.get();- if (logCommit.getCommitDate() != null) {- for (String file: logCommit.getChangedFiles())- files.put(file, logCommit.getCommitDate().getTime());+ if (currentCommit.getCommitDate() != null) {+ for (String file: currentCommit.getChangedFiles())+ files.put(file, currentCommit.getCommitDate().getTime());}- if (logCommit.getCommitter() != null)- users.add(new NameAndEmail(logCommit.getCommitter()));+ if (currentCommit.getCommitter() != null)+ users.add(new NameAndEmail(currentCommit.getCommitter()));- if (logCommit.getAuthor() != null) {- NameAndEmail nameAndEmail = new NameAndEmail(logCommit.getAuthor());+ if (currentCommit.getAuthor() != null) {+ NameAndEmail nameAndEmail = new NameAndEmail(currentCommit.getAuthor());users.add(nameAndEmail);- String emailAddress = logCommit.getAuthor().getEmailAddress();+ String emailAddress = currentCommit.getAuthor().getEmailAddress();if (StringUtils.isNotBlank(emailAddress)) {ByteIterable emailKey = new StringByteIterable(emailAddress);int userIndex = readInt(emailToIndexStore, txn, emailKey, -1);@@ -608,48 +531,30 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager implemenew ArrayByteIterable(SerializationUtils.serialize(nameAndEmail)));}- if (logCommit.getCommitDate() != null && logCommit.getParentHashes().size() <= 1) {- int dayValue = new Day(logCommit.getCommitDate()).getValue();- Map<Integer, Contribution> contributionsOnDay =- dailyContributionsCache.get(dayValue);- if (contributionsOnDay == null) {- contributionsOnDay = deserializeContributions(readBytes(- dailyContributionsStore, txn, new IntByteIterable(dayValue)));- dailyContributionsCache.put(dayValue, contributionsOnDay);- }- updateContribution(contributionsOnDay, userIndex, logCommit);- }-- for (FileChange change: logCommit.getFileChanges()) {- String path = change.getPath();- int pathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,- nextIndex, path);- int edits = change.getAdditions() + change.getDeletions();- if (edits < 0)- edits = 100;- updateEdits(editsStore, txn, editsCache, userIndex, pathIndex, edits);- while (path.contains("/")) {- path = StringUtils.substringBeforeLast(path, "/");- pathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,+ for (FileChange change: currentCommit.getFileChanges()) {+ for (String path: change.getPaths()) {+ int pathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,nextIndex, path);- updateEdits(editsStore, txn, editsCache, userIndex, pathIndex, edits);+ updateCommitCount(commitCountsStore, txn, commitCountCache, userIndex, pathIndex);+ while (path.contains("/")) {+ path = StringUtils.substringBeforeLast(path, "/");+ pathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,+ nextIndex, path);+ updateCommitCount(commitCountsStore, txn, commitCountCache, userIndex, pathIndex);+ }+ pathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,+ nextIndex, "");+ updateCommitCount(commitCountsStore, txn, commitCountCache, userIndex, pathIndex);}- pathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,- nextIndex, "");- updateEdits(editsStore, txn, editsCache, userIndex, pathIndex, edits);}}}- if (logCommit.getCommitDate() != null && logCommit.getParentHashes().size() <= 1) {- int dayValue = new Day(logCommit.getCommitDate()).getValue();- updateContribution(overallContributions, dayValue, logCommit);- }-- for (FileChange change: logCommit.getFileChanges()) {- if (change.getOldPath() != null) {+ for (FileChange change: currentCommit.getFileChanges()) {+ if (change.getOldPath() != null && change.getNewPath() != null+ && !change.getOldPath().equals(change.getNewPath())) {int pathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,- nextIndex, change.getPath());+ nextIndex, change.getNewPath());ByteIterable pathKey = new IntByteIterable(pathIndex);Set<Integer> historyPathIndexes = new HashSet<>();byte[] bytesOfHistoryPaths = readBytes(historyPathsStore, txn, pathKey);@@ -680,103 +585,227 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme}}}- }-- if (logCommit.getCommitDate() != null && logCommit.getParentHashes().size() <= 1) {- int day = new Day(logCommit.getCommitDate()).getValue();-- Map<String, Integer> lineStatsOnDay = lineStats.get(day);- if (lineStatsOnDay == null) {- lineStatsOnDay = new HashMap<>();- lineStats.put(day, lineStatsOnDay);- }-- Map<String, Integer> languageLines = new HashMap<>();- for (FileChange change: logCommit.getFileChanges()) {- int lines = change.getAdditions() - change.getDeletions();- int lastIndexOfDot = change.getPath().lastIndexOf('.');- if (lastIndexOfDot != -1 && lines != 0) {- String fileExt = change.getPath().substring(lastIndexOfDot+1).toLowerCase();- String language = PROGRAMMING_LANGUAGES.get(fileExt);- if (language != null) {- Integer accumulatedLines = languageLines.get(language);- if (accumulatedLines != null)- lines += accumulatedLines;- languageLines.put(language, lines);- }- }- }-- for (Map.Entry<String, Integer> entry: languageLines.entrySet()) {- String language = entry.getKey();- Integer lines = entry.getValue();- Integer accumulatedLines = lineStatsOnDay.get(language);- if (accumulatedLines != null)- lines += accumulatedLines;- lineStatsOnDay.put(language, lines);- }+ }+ }+ }+ }.pump();++ writeInt(defaultStore, txn, COMMIT_COUNT_KEY, totalCommitCount.get());+ totalCommitCountCache.remove(project.getId());++ writeInt(defaultStore, txn, NEXT_USER_INDEX_KEY, nextIndex.user);+ writeInt(defaultStore, txn, NEXT_PATH_INDEX_KEY, nextIndex.path);++ userBytes = SerializationUtils.serialize((Serializable) users);+ defaultStore.put(txn, USERS_KEY, new ArrayByteIterable(userBytes));+ usersCache.remove(project.getId());++ if (files.size() > MAX_COLLECTING_FILES) {+ List<String> fileList = new ArrayList<>(files.keySet());+ fileList.sort((file1, file2)->files.get(file1).compareTo(files.get(file2)));+ for (int i=0; i<fileList.size() - MAX_COLLECTING_FILES; i++)+ files.remove(fileList.get(i));+ }+ fileBytes = SerializationUtils.serialize((Serializable) files);+ defaultStore.put(txn, FILES_KEY, new ArrayByteIterable(fileBytes));+ filesCache.remove(project.getId());++ for (Map.Entry<Long, Integer> entry: commitCountCache.entrySet())+ writeInt(commitCountsStore, txn, new LongByteIterable(entry.getKey()), entry.getValue());++ lastCommitIdRef.set(intermediateCommitId);+ defaultStore.put(txn, LAST_COMMIT_KEY, new CommitByteIterable(lastCommitIdRef.get()));+ }+ });+ }+ }++ if (GitUtils.branch2ref(project.getDefaultBranch()).equals(refName)) {+ ObjectId lastCommitId = env.computeInTransaction(new TransactionalComputable<ObjectId>() {++ @Override+ public ObjectId compute(Transaction txn) {+ byte[] lastCommitBytes = readBytes(defaultStore, txn, LAST_COMMIT_OF_DEFAULT_BRANCH_KEY);+ if (lastCommitBytes != null) {+ ObjectId lastCommitId = ObjectId.fromRaw(lastCommitBytes);+ if (repository.hasObject(lastCommitId)+ && GitUtils.isMergedInto(repository, null, lastCommitId, commitId)) {+ return lastCommitId;+ }+ }+ return null;+ }++ });++ if (lastCommitId == null) {+ Map<Integer, Map<String, Integer>> lineStats = new HashMap<>();+ Map<Integer, Map<Integer, Contribution>> dailyContributionsCache = new HashMap<>();+ Map<Integer, Contribution> overallContributions = new HashMap<>();++ env.executeInTransaction(new TransactionalExecutable() {++ @Override+ public void execute(Transaction txn) {+ new ElementPumper<GitCommit>() {++ @Override+ public void generate(Consumer<GitCommit> consumer) {+ List<String> revisions = new ArrayList<>();+ revisions.add(commitId.name());++ EnumSet<LogCommand.Field> fields = EnumSet.allOf(LogCommand.Field.class);+ fields.remove(LogCommand.Field.SUBJECT);+ fields.remove(LogCommand.Field.BODY);+ fields.remove(LogCommand.Field.COMMITTER);+ fields.remove(LogCommand.Field.PARENTS);+ new LogCommand(project.getGitDir()) {++ @Override+ protected void consume(GitCommit commit) {+ consumer.accept(commit);}- logCommitOptional = queue.take();- }- if (logException.get() != null)- throw logException.get();- } catch (Exception e) {- throw ExceptionUtils.unchecked(e);- }-- for (Map.Entry<Integer, Map<Integer, Contribution>> entry: dailyContributionsCache.entrySet()) {- byte[] bytesOfContributionsOnDay = serializeContributions(entry.getValue());- dailyContributionsStore.put(txn, new IntByteIterable(entry.getKey()),- new ArrayByteIterable(bytesOfContributionsOnDay));+ }.firstParent(true).revisions(revisions).fields(fields).call();}- defaultStore.put(txn, OVERALL_CONTRIBUTIONS_KEY,- new ArrayByteIterable(serializeContributions(overallContributions)));-- bytesOfLineStats = SerializationUtils.serialize((Serializable) lineStats);- defaultStore.put(txn, LINE_STATS_KEY, new ArrayByteIterable(bytesOfLineStats));-- writeInt(defaultStore, txn, NEXT_USER_INDEX_KEY, nextIndex.user);- writeInt(defaultStore, txn, NEXT_PATH_INDEX_KEY, nextIndex.path);-- userBytes = SerializationUtils.serialize((Serializable) users);- defaultStore.put(txn, USERS_KEY, new ArrayByteIterable(userBytes));- usersCache.remove(project.getId());-- if (files.size() > MAX_COLLECTING_FILES) {- List<String> fileList = new ArrayList<>(files.keySet());- fileList.sort((file1, file2)->files.get(file1).compareTo(files.get(file2)));- for (int i=0; i<fileList.size() - MAX_COLLECTING_FILES; i++)- files.remove(fileList.get(i));++ @Override+ public void process(GitCommit currentCommit) {+ updateOverallContributionsAndLineStats(txn, emailToIndexStore,+ currentCommit, lineStats, overallContributions);+ int dayValue = new Day(currentCommit.getCommitDate()).getValue();+ Map<Integer, Contribution> contributionsOnDay = dailyContributionsCache.get(dayValue);+ if (contributionsOnDay == null) {+ contributionsOnDay = new HashMap<>();+ dailyContributionsCache.put(dayValue, contributionsOnDay);+ }+ updateContribution(txn, emailToIndexStore, currentCommit, contributionsOnDay);}- fileBytes = SerializationUtils.serialize((Serializable) files);- defaultStore.put(txn, FILES_KEY, new ArrayByteIterable(fileBytes));- filesCache.remove(project.getId());-- for (Map.Entry<Long, Integer> entry: editsCache.entrySet())- writeInt(editsStore, txn, new LongByteIterable(entry.getKey()), entry.getValue());- defaultStore.put(txn, LAST_COMMIT_OF_DEFAULT_BRANCH_KEY, new CommitByteIterable(currentCommitId));+ }.pump();++ for (int dayValue: deserializeContributions(readBytes(defaultStore, txn, OVERALL_CONTRIBUTIONS_KEY)).keySet())+ dailyContributionsStore.delete(txn, new IntByteIterable(dayValue));+ for (Map.Entry<Integer, Map<Integer, Contribution>> entry: dailyContributionsCache.entrySet()) {+ byte[] bytesOfContributionsOnDay = serializeContributions(entry.getValue());+ dailyContributionsStore.put(txn, new IntByteIterable(entry.getKey()),+ new ArrayByteIterable(bytesOfContributionsOnDay));}- });- }- }+ defaultStore.put(txn, OVERALL_CONTRIBUTIONS_KEY,+ new ArrayByteIterable(serializeContributions(overallContributions)));++ byte[] bytesOfLineStats = SerializationUtils.serialize((Serializable) lineStats);+ defaultStore.put(txn, LINE_STATS_KEY, new ArrayByteIterable(bytesOfLineStats));++ defaultStore.put(txn, LAST_COMMIT_OF_DEFAULT_BRANCH_KEY, new CommitByteIterable(commitId));+ }++ });+ } else {+ env.executeInTransaction(new TransactionalExecutable() {++ @SuppressWarnings("unchecked")+ @Override+ public void execute(Transaction txn) {+ Map<Integer, Map<String, Integer>> lineStats;+ byte[] bytesOfLineStats = readBytes(defaultStore, txn, LINE_STATS_KEY);+ if (bytesOfLineStats != null) {+ lineStats = (Map<Integer, Map<String, Integer>>) SerializationUtils.deserialize(+ bytesOfLineStats);+ } else {+ lineStats = new HashMap<>();+ }++ ListNumStatsCommand command = new ListNumStatsCommand(project.getGitDir());+ List<FileChange> fileChanges = command.fromRev(lastCommitId.name()).toRev(commitId.name()).call();+ RevCommit revCommit = project.getRevCommit(commitId, true);+ GitCommit gitCommit = new GitCommit(revCommit.name(), null, null, revCommit.getAuthorIdent(),+ revCommit.getCommitterIdent().getWhen(), null, null, fileChanges);++ int dayValue = new Day(revCommit.getCommitterIdent().getWhen()).getValue();+ Map<Integer, Contribution> contributionsOnDay = deserializeContributions(+ readBytes(dailyContributionsStore, txn, new IntByteIterable(dayValue)));+ Map<Integer, Contribution> overallContributions =+ deserializeContributions(readBytes(defaultStore, txn, OVERALL_CONTRIBUTIONS_KEY));++ updateOverallContributionsAndLineStats(txn, emailToIndexStore, gitCommit, lineStats, overallContributions);+ updateContribution(txn, emailToIndexStore, gitCommit, contributionsOnDay);++ byte[] bytesOfContributionsOnDay = serializeContributions(contributionsOnDay);+ dailyContributionsStore.put(txn, new IntByteIterable(dayValue),+ new ArrayByteIterable(bytesOfContributionsOnDay));+ defaultStore.put(txn, OVERALL_CONTRIBUTIONS_KEY,+ new ArrayByteIterable(serializeContributions(overallContributions)));++ bytesOfLineStats = SerializationUtils.serialize((Serializable) lineStats);+ defaultStore.put(txn, LINE_STATS_KEY, new ArrayByteIterable(bytesOfLineStats));++ defaultStore.put(txn, LAST_COMMIT_OF_DEFAULT_BRANCH_KEY, new CommitByteIterable(commitId));+ }++ });++ }+ }++ for (Long issueNumber: fixedIssueNumbers) {+ Issue issue = issueManager.find(project, issueNumber);+ if (issue != null)+ listenerRegistry.post(new IssueCommitted(issue));}+logger.debug("Collected commit information (project: {}, ref: {})", project.getName(), refName);}- private void updateContribution(Map<Integer, Contribution> contributions, int key, GitCommit commit) {- Contribution contribution = contributions.get(key);- if (contribution != null) {- contribution = new Contribution(- contribution.getCommits()+1,- contribution.getAdditions()+commit.getAdditions(),- contribution.getDeletions()+commit.getDeletions());- } else {- contribution = new Contribution(1, commit.getAdditions(), commit.getDeletions());+ private void updateContribution(Transaction txn, Store emailToIndexStore,+ GitCommit currentCommit, Map<Integer, Contribution> contributionsOnDay) {+ if (currentCommit.getAuthor() != null+ && StringUtils.isNotBlank(currentCommit.getAuthor().getEmailAddress())) {+ String emailAddress = currentCommit.getAuthor().getEmailAddress();+ ByteIterable emailKey = new StringByteIterable(emailAddress);+ int userIndex = readInt(emailToIndexStore, txn, emailKey, -1);+ Preconditions.checkState(userIndex != -1);+ updateContribution(contributionsOnDay, userIndex, currentCommit);+ }+ }++ private void updateOverallContributionsAndLineStats(Transaction txn, Store emailToIndexStore,+ GitCommit currentCommit, Map<Integer, Map<String, Integer>> lineStats,+ Map<Integer, Contribution> overallContributions) {+ int dayValue = new Day(currentCommit.getCommitDate()).getValue();+ updateContribution(overallContributions, dayValue, currentCommit);++ Map<String, Integer> lineStatsOnDay = lineStats.get(dayValue);+ if (lineStatsOnDay == null) {+ lineStatsOnDay = new HashMap<>();+ lineStats.put(dayValue, lineStatsOnDay);+ }++ Map<String, Integer> languageLines = new HashMap<>();+ for (FileChange change: currentCommit.getFileChanges()) {+ int lines = change.getAdditions() - change.getDeletions();+ int lastIndexOfDot = change.getNewPath().lastIndexOf('.');+ if (lastIndexOfDot != -1 && lines != 0) {+ String fileExt = change.getNewPath().substring(lastIndexOfDot+1).toLowerCase();+ String language = PROGRAMMING_LANGUAGES.get(fileExt);+ if (language != null) {+ Integer accumulatedLines = languageLines.get(language);+ if (accumulatedLines != null)+ lines += accumulatedLines;+ languageLines.put(language, lines);+ }+ }+ }++ for (Map.Entry<String, Integer> entry: languageLines.entrySet()) {+ String language = entry.getKey();+ Integer lines = entry.getValue();+ Integer accumulatedLines = lineStatsOnDay.get(language);+ if (accumulatedLines != null)+ lines += accumulatedLines;+ lineStatsOnDay.put(language, lines);}- contributions.put(key, contribution);}private int getPathIndex(Store pathToIndexStore, Store indexToPathStore, Transaction txn,@@ -791,15 +820,15 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager implemereturn pathIndex;}- private void updateEdits(Store store, Transaction txn,- Map<Long, Integer> editsCache, int userIndex, int pathIndex, int edits) {- long editsKey = (userIndex<<32)|pathIndex;+ private void updateCommitCount(Store store, Transaction txn,+ Map<Long, Integer> commitCountCache, int userIndex, int pathIndex) {+ long commitCountKey = (userIndex<<32)|pathIndex;- Integer editsOfPathByUser = editsCache.get(editsKey);- if (editsOfPathByUser == null)- editsOfPathByUser = readInt(store, txn, new LongByteIterable(editsKey), 0);- editsOfPathByUser += edits;- editsCache.put(editsKey, editsOfPathByUser);+ Integer commitCountOfPathByUser = commitCountCache.get(commitCountKey);+ if (commitCountOfPathByUser == null)+ commitCountOfPathByUser = readInt(store, txn, new LongByteIterable(commitCountKey), 0);+ commitCountOfPathByUser ++;+ commitCountCache.put(commitCountKey, commitCountOfPathByUser);}@Override@@ -895,13 +924,26 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme});}+ private void updateContribution(Map<Integer, Contribution> contributions, int key, GitCommit commit) {+ Contribution contribution = contributions.get(key);+ if (contribution != null) {+ contribution = new Contribution(+ contribution.getCommits()+1,+ contribution.getAdditions()+commit.getAdditions(),+ contribution.getDeletions()+commit.getDeletions());+ } else {+ contribution = new Contribution(1, commit.getAdditions(), commit.getDeletions());+ }+ contributions.put(key, contribution);+ }+@Override- public int getEdits(Project project, User user, String path) {+ public int getCommitCount(Project project, User user, String path) {if (user.getEmail() != null) {Environment env = getEnv(project.getId().toString());Store emailToIndexStore = getStore(env, EMAIL_TO_INDEX_STORE);Store pathToIndexStore = getStore(env, PATH_TO_INDEX_STORE);- Store editsStore = getStore(env, EDITS_STORE);+ Store commitCountStore = getStore(env, COMMIT_COUNTS_STORE);return env.computeInReadonlyTransaction(new TransactionalComputable<Integer>() {@Override@@ -910,8 +952,8 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager implemeif (userIndex != -1) {int pathIndex = readInt(pathToIndexStore, txn, new StringByteIterable(path), -1);if (pathIndex != -1) {- long editsKey = (userIndex<<32)|pathIndex;- return readInt(editsStore, txn, new LongByteIterable(editsKey), 0);+ long commitCountKey = (userIndex<<32)|pathIndex;+ return readInt(commitCountStore, txn, new LongByteIterable(commitCountKey), 0);}}return 0;@@ -973,7 +1015,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager implemeLong projectId = event.getEntity().getId();removeEnv(projectId.toString());filesCache.remove(projectId);- commitCountCache.remove(projectId);+ totalCommitCountCache.remove(projectId);usersCache.remove(projectId);}}@@ -1053,7 +1095,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme@Sessional@Overridepublic int getCommitCount(Project project) {- Integer commitCount = commitCountCache.get(project.getId());+ Integer commitCount = totalCommitCountCache.get(project.getId());if (commitCount == null) {Environment env = getEnv(project.getId().toString());Store store = getStore(env, DEFAULT_STORE);@@ -1065,7 +1107,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager implemereturn readInt(store, txn, COMMIT_COUNT_KEY, 0);}});- commitCountCache.put(project.getId(), commitCount);+ totalCommitCountCache.put(project.getId(), commitCount);}return commitCount;}@@ -1356,12 +1398,6 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager implemereturn INFO_VERSION;}- private static class NextIndex {- int user;-- int path;- }-@Overridepublic Collection<ObjectId> getFixCommits(Project project, Long issueNumber) {Environment env = getEnv(project.getId().toString());@@ -1378,4 +1414,10 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme}+ private static class NextIndex {+ int user;++ int path;+ }+}diff --git a/server-core/src/main/java/io/onedev/server/maintenance/DefaultDataManager.java b/server-core/src/main/java/io/onedev/server/maintenance/DefaultDataManager.javaindex 7ea0f42f0d..3a3ad9c561 100644--- a/server-core/src/main/java/io/onedev/server/maintenance/DefaultDataManager.java+++ b/server-core/src/main/java/io/onedev/server/maintenance/DefaultDataManager.java@@ -163,7 +163,7 @@ public class DefaultDataManager implements DataManager, Serializable {setting = settingManager.getSetting(Key.JOB_EXECUTORS);if (setting == null) {AutoDiscoveredJobExecutor executor = new AutoDiscoveredJobExecutor();- executor.setName("auto discovered");+ executor.setName("auto-discovered");settingManager.saveJobExecutors(Lists.newArrayList(executor));}setting = settingManager.getSetting(Key.JOB_SCRIPTS);diff --git a/server-core/src/main/java/io/onedev/server/model/Project.java b/server-core/src/main/java/io/onedev/server/model/Project.javaindex 019cfb2654..3bb4b3886a 100644--- a/server-core/src/main/java/io/onedev/server/model/Project.java+++ b/server-core/src/main/java/io/onedev/server/model/Project.java@@ -557,31 +557,38 @@ public class Project extends AbstractEntity {Optional<Blob> blob = getBlobCache().get(blobIdent);if (blob == null) {try (RevWalk revWalk = new RevWalk(getRepository())) {- ObjectId revId = getObjectId(blobIdent.revision, true);- RevTree revTree = revWalk.parseCommit(revId).getTree();- TreeWalk treeWalk = TreeWalk.forPath(getRepository(), blobIdent.path, revTree);- if (treeWalk != null) {- ObjectId blobId = treeWalk.getObjectId(0);- if (blobIdent.isGitLink()) {- String url = getSubmodules(blobIdent.revision).get(blobIdent.path);- if (url == null) {- if (mustExist)- throw new ObjectNotFoundException("Unable to find submodule '" + blobIdent.path + "' in .gitmodules");- else- blob = Optional.absent();- } else {- String hash = blobId.name();- blob = Optional.of(new Blob(blobIdent, blobId, new Submodule(url, hash).toString().getBytes()));- }- } else if (blobIdent.isTree()) {- throw new NotFileException("Path '" + blobIdent.path + "' is a tree");- } else {- blob = Optional.of(new Blob(blobIdent, blobId, treeWalk.getObjectReader()));- }- } else if (mustExist) {- throw new ObjectNotFoundException("Unable to find blob path '" + blobIdent.path + "' in revision '" + blobIdent.revision + "'");- } else {- blob = Optional.absent();+ ObjectId revId = getObjectId(blobIdent.revision, mustExist);+ if (revId != null) {+ RevCommit commit = GitUtils.parseCommit(revWalk, revId);+ if (commit != null) {+ RevTree revTree = commit.getTree();+ TreeWalk treeWalk = TreeWalk.forPath(getRepository(), blobIdent.path, revTree);+ if (treeWalk != null) {+ ObjectId blobId = treeWalk.getObjectId(0);+ if (blobIdent.isGitLink()) {+ String url = getSubmodules(blobIdent.revision).get(blobIdent.path);+ if (url == null) {+ if (mustExist)+ throw new ObjectNotFoundException("Unable to find submodule '" + blobIdent.path + "' in .gitmodules");+ else+ blob = Optional.absent();+ } else {+ String hash = blobId.name();+ blob = Optional.of(new Blob(blobIdent, blobId, new Submodule(url, hash).toString().getBytes()));+ }+ } else if (blobIdent.isTree()) {+ throw new NotFileException("Path '" + blobIdent.path + "' is a tree");+ } else {+ blob = Optional.of(new Blob(blobIdent, blobId, treeWalk.getObjectReader()));+ }+ }+ }+ }+ if (blob == null) {+ if (mustExist)+ throw new ObjectNotFoundException("Unable to find blob ident: " + blobIdent);+ else+ blob = Optional.absent();}getBlobCache().put(blobIdent, blob);} catch (IOException e) {diff --git a/server-core/src/main/java/io/onedev/server/model/PullRequestUpdate.java b/server-core/src/main/java/io/onedev/server/model/PullRequestUpdate.javaindex 9072627dc6..f245f186c1 100644--- a/server-core/src/main/java/io/onedev/server/model/PullRequestUpdate.java+++ b/server-core/src/main/java/io/onedev/server/model/PullRequestUpdate.java@@ -31,8 +31,6 @@ import com.google.common.base.Preconditions;import io.onedev.server.entitymanager.impl.DefaultCodeCommentRelationManager;import io.onedev.server.git.GitUtils;-import io.onedev.server.git.command.FileChange;-import io.onedev.server.git.command.ListFileChangesCommand;@Entity@Table(indexes={@Index(columnList="o_request_id"), @Index(columnList="date")})@@ -59,8 +57,6 @@ public class PullRequestUpdate extends AbstractEntity {private transient Collection<String> changedFiles;- private transient Collection<FileChange> fileChanges;-public PullRequest getRequest() {return request;}@@ -103,16 +99,6 @@ public class PullRequestUpdate extends AbstractEntity {GitUtils.deleteRef(GitUtils.getRefUpdate(getRequest().getTargetProject().getRepository(), getHeadRef()));}- public Collection<FileChange> getFileChanges() {- if (fileChanges == null) {- ListFileChangesCommand cmd = new ListFileChangesCommand(getRequest().getTargetProject().getGitDir());- cmd.fromRev(getBaseCommitHash());- cmd.toRev(getHeadCommitHash());- fileChanges = cmd.call();- }- return fileChanges;- }-/*** Get changed files of this update since previous update. This calculation* excludes changes introduced by commits from target branch (this willdiff --git a/server-core/src/main/java/io/onedev/server/notification/CommitNotificationManager.java b/server-core/src/main/java/io/onedev/server/notification/CommitNotificationManager.javaindex 012ef2c2c7..928ef10122 100644--- a/server-core/src/main/java/io/onedev/server/notification/CommitNotificationManager.java+++ b/server-core/src/main/java/io/onedev/server/notification/CommitNotificationManager.java@@ -84,11 +84,13 @@ public class CommitNotificationManager {}}- RevCommit commit = project.getRevCommit(event.getNewCommitId(), true);- String subject = String.format("Subscribed commit at ref '%s': %s", event.getRefName(), commit.getShortMessage());- String url = urlManager.urlFor(project, commit);- String body = String.format("Visit <a href='%s'>%s</a> for details", url, url);- mailManager.sendMailAsync(notifyEmails, subject, body.toString());+ RevCommit commit = project.getRevCommit(event.getNewCommitId(), false);+ if (commit != null) {+ String subject = String.format("Subscribed commit at ref '%s': %s", event.getRefName(), commit.getShortMessage());+ String url = urlManager.urlFor(project, commit);+ String body = String.format("Visit <a href='%s'>%s</a> for details", url, url);+ mailManager.sendMailAsync(notifyEmails, subject, body.toString());+ }}}}diff --git a/server-core/src/main/java/io/onedev/server/util/ElementPumper.java b/server-core/src/main/java/io/onedev/server/util/ElementPumper.javanew file mode 100644index 0000000000..338fdf45e2--- /dev/null+++ b/server-core/src/main/java/io/onedev/server/util/ElementPumper.java@@ -0,0 +1,74 @@+package io.onedev.server.util;++import java.util.Optional;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.SynchronousQueue;+import java.util.concurrent.atomic.AtomicReference;+import java.util.function.Consumer;++import io.onedev.commons.utils.ExceptionUtils;+import io.onedev.server.OneDev;++/**+ * A pumper generates elements in a different thread, and consumes the element+ * in current thread. It is designed as some scenario requires that elements+ * should be consumed in current thread, and collecting all generated elements+ * into a collection before processing is not acceptable due to memory constraint+ *+ * @author robin+ *+ * @param <T>+ */+public abstract class ElementPumper<T> {++ public void pump() {+ SynchronousQueue<Optional<T>> queue = new SynchronousQueue<>();+ AtomicReference<Exception> exceptionRef = new AtomicReference<>(null);++ OneDev.getInstance(ExecutorService.class).execute(new Runnable() {++ @Override+ public void run() {+ try {+ generate(new Consumer<T>() {++ @Override+ public void accept(T t) {+ try {+ queue.put(Optional.of(t));+ } catch (InterruptedException e) {+ throw new RuntimeException(e);+ }+ }++ });+ } catch (Exception e) {+ exceptionRef.set(e);+ } finally {+ try {+ queue.put(Optional.empty());+ } catch (InterruptedException e) {+ }+ }+ }++ });++ try {+ Optional<T> element = queue.take();+ while (element.isPresent()) {+ process(element.get());+ element = queue.take();+ }+ if (exceptionRef.get() != null)+ throw exceptionRef.get();+ } catch (Exception e) {+ throw ExceptionUtils.unchecked(e);+ }+ }++ public abstract void generate(Consumer<T> consumer);++ public abstract void process(T element);++}diff --git a/server-core/src/main/java/io/onedev/server/web/component/diff/revision/RevisionDiffPanel.html b/server-core/src/main/java/io/onedev/server/web/component/diff/revision/RevisionDiffPanel.htmlindex fda557ffa4..fc716ead70 100644--- a/server-core/src/main/java/io/onedev/server/web/component/diff/revision/RevisionDiffPanel.html+++ b/server-core/src/main/java/io/onedev/server/web/component/diff/revision/RevisionDiffPanel.html@@ -15,10 +15,10 @@<td class="whitespace-option"><span wicket:id="whitespaceOption" class="btn btn-default" title="White space options">White space <i class="caret"></i></span></td>- <td class="path-filter">- <form wicket:id="pathFilter">+ <td class="path-filter">+ <form wicket:id="pathFilter" class="clearable-wrapper"><div class="input-group">- <input wicket:id="input" class="form-control no-autofocus" spellcheck="false" autocomplete="off" placeholder="Filter by path" type="text">+ <input wicket:id="input" class="form-control no-autofocus" spellcheck="false" autocomplete="off" placeholder="Filter by path"><span class="input-group-btn"><button wicket:id="submit" title="Filter" class="btn btn-default" type="submit"><i class="fa fa-search"></i></button></span>diff --git a/server-core/src/test/java/io/onedev/server/git/command/LogCommandTest.java b/server-core/src/test/java/io/onedev/server/git/command/LogCommandTest.javaindex 022de7a836..993659af90 100644--- a/server-core/src/test/java/io/onedev/server/git/command/LogCommandTest.java+++ b/server-core/src/test/java/io/onedev/server/git/command/LogCommandTest.java@@ -3,6 +3,7 @@ package io.onedev.server.git.command;import static org.junit.Assert.assertEquals;import java.util.ArrayList;+import java.util.EnumSet;import java.util.List;import org.junit.Test;@@ -37,7 +38,7 @@ public class LogCommandTest extends AbstractGitTest {+ "7th line\n"+ "8th line\n"+ "9th line\n");- commit("初始commit");+ commit("初始commit\n\nfirst line\nsecond line");rm("测试文件");addFile("改名后 测试文件", ""+ "1st line\n"@@ -67,10 +68,12 @@ public class LogCommandTest extends AbstractGitTest {+ "7th line\n"+ "8th line\n"+ "9th line\n");- commit("第二个commit");+ commit("第二个commit\n\nfirst line\n\nsecond line\nthird line\n");List<GitCommit> commits = new ArrayList<>();-++ EnumSet<LogCommand.Field> fields = EnumSet.allOf(LogCommand.Field.class);+ fields.remove(LogCommand.Field.LINE_CHANGES);new LogCommand(git.getRepository().getDirectory()) {@Override@@ -78,24 +81,47 @@ public class LogCommandTest extends AbstractGitTest {commits.add(commit);}- }.revisions(Lists.newArrayList("master")).call();+ }.revisions(Lists.newArrayList("master")).fields(fields).call();assertEquals(2, commits.size());+ GitCommit commit1 = commits.get(1);+ assertEquals("初始commit", commit1.getSubject());+ assertEquals("first line\nsecond line", commit1.getBody());++ GitCommit commit2 = commits.get(0);+ assertEquals(3, commit2.getFileChanges().size());+ assertEquals("另一个 测试文件", commit2.getFileChanges().get(0).getNewPath());+ assertEquals("改名后 测试文件", commit2.getFileChanges().get(1).getNewPath());+ assertEquals("测试文件", commit2.getFileChanges().get(1).getOldPath());+ assertEquals("第三个 测试文件", commit2.getFileChanges().get(2).getNewPath());+ assertEquals("第二个commit", commit2.getSubject());+ assertEquals("first line\n\nsecond line\nthird line", commit2.getBody());++ commits.clear();+ fields = EnumSet.allOf(LogCommand.Field.class);+ new LogCommand(git.getRepository().getDirectory()) {++ @Override+ protected void consume(GitCommit commit) {+ commits.add(commit);+ }++ }.revisions(Lists.newArrayList("master")).fields(fields).call();+GitCommit commit = commits.get(0);assertEquals(3, commit.getFileChanges().size());-- assertEquals("另一个 测试文件", commit.getFileChanges().get(0).getPath());+ assertEquals("另一个 测试文件", commit.getFileChanges().get(0).getNewPath());assertEquals(0, commit.getFileChanges().get(0).getAdditions());assertEquals(1, commit.getFileChanges().get(0).getDeletions());- assertEquals("改名后 测试文件", commit.getFileChanges().get(1).getPath());+ assertEquals("改名后 测试文件", commit.getFileChanges().get(1).getNewPath());assertEquals("测试文件", commit.getFileChanges().get(1).getOldPath());- assertEquals("第三个 测试文件", commit.getFileChanges().get(2).getPath());+ assertEquals("第三个 测试文件", commit.getFileChanges().get(2).getNewPath());assertEquals(8, commit.getFileChanges().get(2).getAdditions());- assertEquals(0, commit.getFileChanges().get(2).getDeletions());+ assertEquals(0, commit.getFileChanges().get(2).getDeletions());}}\ No newline at end of file