The Unified and Autonomous Development Platform
Some performance improvements and bug fixes
robin shen2ffa9028c9ab
diff --git a/core/src/main/java/com/gitplex/server/git/Blame.java b/core/src/main/java/com/gitplex/server/git/Blame.javaindex d4a421c1c4..92f889f137 100644--- a/core/src/main/java/com/gitplex/server/git/Blame.java+++ b/core/src/main/java/com/gitplex/server/git/Blame.java@@ -8,16 +8,16 @@ import com.gitplex.jsymbol.Range;public class Blame {- private final Commit commit;+ private final BlameCommit commit;private final List<Range> ranges;- public Blame(Commit commit, List<Range> ranges) {+ public Blame(BlameCommit commit, List<Range> ranges) {this.commit = commit;this.ranges = ranges;}- public Commit getCommit() {+ public BlameCommit getCommit() {return commit;}diff --git a/core/src/main/java/com/gitplex/server/git/Commit.java b/core/src/main/java/com/gitplex/server/git/BlameCommit.javasimilarity index 83%rename from core/src/main/java/com/gitplex/server/git/Commit.javarename to core/src/main/java/com/gitplex/server/git/BlameCommit.javaindex b24bd1330c..f569288c83 100644--- a/core/src/main/java/com/gitplex/server/git/Commit.java+++ b/core/src/main/java/com/gitplex/server/git/BlameCommit.java@@ -9,7 +9,7 @@ import org.eclipse.jgit.lib.PersonIdent;import com.google.common.base.Objects;-public class Commit implements Serializable {+public class BlameCommit implements Serializable {private static final long serialVersionUID = 1L;@@ -21,14 +21,14 @@ public class Commit implements Serializable {private final PersonIdent author;- public Commit(String hash, PersonIdent committer, PersonIdent author, String subject) {+ public BlameCommit(String hash, PersonIdent committer, PersonIdent author, String subject) {this.hash = checkNotNull(hash, "hash");this.committer = checkNotNull(committer, "committer");this.author = checkNotNull(author, "author");this.subject = checkNotNull(subject, "subject");}- public Commit(Commit commit) {+ public BlameCommit(BlameCommit commit) {this(commit.getHash(), commit.getCommitter(), commit.getAuthor(), commit.getSubject());}@@ -56,9 +56,9 @@ public class Commit implements Serializable {@Overridepublic boolean equals(Object other) {if (this == other) return true;- if (!(other instanceof Commit)) return false;+ if (!(other instanceof BlameCommit)) return false;- Commit rhs = (Commit) other;+ BlameCommit rhs = (BlameCommit) other;return Objects.equal(hash, rhs.hash);}@@ -90,10 +90,10 @@ public class Commit implements Serializable {public String subject;- public Commit build() {+ public BlameCommit build() {PersonIdent committer = GitUtils.newPersonIdent(committerName, committerEmail, committerDate);PersonIdent author = GitUtils.newPersonIdent(authorName, authorEmail, authorDate);- return new Commit(hash, committer, author, subject.trim());+ return new BlameCommit(hash, committer, author, subject.trim());}}}diff --git a/core/src/main/java/com/gitplex/server/git/command/BlameCommand.java b/core/src/main/java/com/gitplex/server/git/command/BlameCommand.javaindex 6d222dd546..182416bdd8 100644--- a/core/src/main/java/com/gitplex/server/git/command/BlameCommand.java+++ b/core/src/main/java/com/gitplex/server/git/command/BlameCommand.java@@ -17,7 +17,7 @@ import org.slf4j.LoggerFactory;import com.gitplex.jsymbol.Range;import com.gitplex.server.git.Blame;-import com.gitplex.server.git.Commit;+import com.gitplex.server.git.BlameCommit;import com.gitplex.server.git.GitUtils;import com.gitplex.server.util.execution.Commandline;import com.gitplex.server.util.execution.ExecuteResult;@@ -71,10 +71,10 @@ public class BlameCommand extends GitCommand<Map<String, Blame>> {Commandline cmd = buildCmd();final Map<String, Blame> blames = new HashMap<>();- final Map<String, Commit> commitMap = new HashMap<>();+ final Map<String, BlameCommit> commitMap = new HashMap<>();- final AtomicReference<Commit> commitRef = new AtomicReference<>(null);- final BriefCommitBuilder commitBuilder = new BriefCommitBuilder();+ final AtomicReference<BlameCommit> commitRef = new AtomicReference<>(null);+ final CommitBuilder commitBuilder = new CommitBuilder();final AtomicBoolean endOfFile = new AtomicBoolean(false);final AtomicInteger beginLine = new AtomicInteger(0);@@ -93,7 +93,7 @@ public class BlameCommand extends GitCommand<Map<String, Blame>> {commitBuilder.hash = null;} else if (commitBuilder.hash == null) {commitBuilder.hash = StringUtils.substringBefore(line, " ");- Commit commit = commitRef.get();+ BlameCommit commit = commitRef.get();if (commit != null && !commitBuilder.hash.equals(commit.getHash())) {Blame blame = blames.get(commit.getHash());if (blame == null) {@@ -142,7 +142,7 @@ public class BlameCommand extends GitCommand<Map<String, Blame>> {result.checkReturnCode();if (endLine.get() > beginLine.get()) {- Commit commit = commitRef.get();+ BlameCommit commit = commitRef.get();Blame blame = blames.get(commit.getHash());if (blame == null) {blame = new Blame(commit, new ArrayList<Range>());@@ -157,7 +157,7 @@ public class BlameCommand extends GitCommand<Map<String, Blame>> {return blames;}- private static class BriefCommitBuilder {+ private static class CommitBuilder {private Date committerDate;@@ -175,8 +175,8 @@ public class BlameCommand extends GitCommand<Map<String, Blame>> {private String summary;- private Commit build() {- return new Commit(+ private BlameCommit build() {+ return new BlameCommit(hash,GitUtils.newPersonIdent(committer, committerEmail, committerDate),GitUtils.newPersonIdent(author, authorEmail, authorDate),diff --git a/core/src/main/java/com/gitplex/server/git/command/LogCommand.java b/core/src/main/java/com/gitplex/server/git/command/LogCommand.javanew file mode 100644index 0000000000..7ea198bb5c--- /dev/null+++ b/core/src/main/java/com/gitplex/server/git/command/LogCommand.java@@ -0,0 +1,117 @@+package com.gitplex.server.git.command;++import java.io.File;+import java.util.ArrayList;+import java.util.List;+import java.util.concurrent.atomic.AtomicBoolean;++import org.apache.commons.lang3.StringUtils;+import org.eclipse.jgit.util.QuotedString;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++import com.gitplex.server.git.GitUtils;+import com.gitplex.server.util.execution.Commandline;+import com.gitplex.server.util.execution.LineConsumer;++public abstract class LogCommand extends GitCommand<Void> {++ private static final Logger logger = LoggerFactory.getLogger(LogCommand.class);++ private List<String> revisions = new ArrayList<>();++ public LogCommand(File gitDir) {+ super(gitDir);+ }++ public List<String> revisions() {+ return revisions;+ }++ public LogCommand revisions(List<String> revisions) {+ this.revisions = revisions;+ return this;+ }++ @Override+ public Void call() {+ Commandline cmd = cmd();+ cmd.addArgs("log",+ "--format=*** commit_begin ***%nhash:%H%nauthor:%aN%nauthorEmail:%aE%n"+ + "committerEmail:%cE%ncommitter:%cN%nparents:%P%ncommitterDate:%cd %n"+ + "authorDate:%ad %n",+ "--date=raw");++ if (!revisions.isEmpty()) {+ for (String revision: revisions)+ cmd.addArgs(revision);+ } else {+ cmd.addArgs("--branches");+ }++ cmd.addArgs("--name-only", "--no-renames");++ LogCommit.Builder commitBuilder = new LogCommit.Builder();++ AtomicBoolean changedFilesBlock = new AtomicBoolean();++ cmd.execute(new LineConsumer() {++ @Override+ public void consume(String line) {+ if (line.equals("*** commit_begin ***")) {+ if (commitBuilder.hash!= null)+ LogCommand.this.consume(commitBuilder.build());+ commitBuilder.hash = null;+ commitBuilder.committerName = null;+ commitBuilder.committerEmail = null;+ commitBuilder.committerDate = null;+ commitBuilder.authorName = null;+ commitBuilder.authorEmail = null;+ commitBuilder.authorDate = null;+ commitBuilder.parentHashes = new ArrayList<>();+ commitBuilder.changedFiles = new ArrayList<>();+ changedFilesBlock.set(false);+ } else if (changedFilesBlock.get()) {+ if (line.trim().length() != 0 && commitBuilder.changedFiles != null)+ commitBuilder.changedFiles.add(QuotedString.GIT_PATH.dequote(line));+ } else if (line.startsWith("hash:")) {+ commitBuilder.hash = line.substring("hash:".length());+ } else if (line.startsWith("author:")) {+ commitBuilder.authorName = line.substring("author:".length());+ } else if (line.startsWith("committer:")) {+ commitBuilder.committerName = line.substring("committer:".length());+ } else if (line.startsWith("authorEmail:")) {+ commitBuilder.authorEmail = line.substring("authorEmail:".length());+ } else if (line.startsWith("committerEmail:")) {+ commitBuilder.committerEmail = line.substring("committerEmail:".length());+ } else if (line.startsWith("parents:")) {+ for (String each: StringUtils.split(line.substring("parents:".length()), " "))+ commitBuilder.parentHashes.add(each);+ } else if (line.startsWith("committerDate:")) {+ commitBuilder.committerDate = GitUtils.parseRawDate(line.substring("committerDate:".length()).trim());+ } else if (line.startsWith("authorDate:")) {+ commitBuilder.authorDate = GitUtils.parseRawDate(line.substring("authorDate:".length()).trim());+ if (commitBuilder.changedFiles != null)+ changedFilesBlock.set(true);+ }+ }++ }, new LineConsumer() {++ @Override+ public void consume(String line) {+ logger.error(line);+ }++ }).checkReturnCode();++ if (commitBuilder.hash != null)+ consume(commitBuilder.build());++ return null;+ }++ protected abstract void consume(LogCommit commit);++}diff --git a/core/src/main/java/com/gitplex/server/git/command/LogCommit.java b/core/src/main/java/com/gitplex/server/git/command/LogCommit.javanew file mode 100644index 0000000000..e2fdf4204a--- /dev/null+++ b/core/src/main/java/com/gitplex/server/git/command/LogCommit.java@@ -0,0 +1,101 @@+package com.gitplex.server.git.command;++import java.io.Serializable;+import java.util.ArrayList;+import java.util.Date;+import java.util.List;++import javax.annotation.Nullable;++import org.apache.commons.lang3.StringUtils;+import org.eclipse.jgit.lib.PersonIdent;++import com.gitplex.server.git.GitUtils;++public class LogCommit implements Serializable {++ private static final long serialVersionUID = 1L;++ private final String hash;++ private final PersonIdent committer;++ private final PersonIdent author;++ private final List<String> parentHashes;++ private final List<String> changedFiles;++ public LogCommit(String hash, @Nullable PersonIdent committer, @Nullable PersonIdent author,+ List<String> parentHashes, List<String> changedFiles) {+ this.hash = hash;+ this.committer = committer;+ this.author = author;+ this.parentHashes = parentHashes;+ this.changedFiles = changedFiles;+ }++ public List<String> getParentHashes() {+ return parentHashes;+ }++ public List<String> getChangedFiles() {+ return changedFiles;+ }++ public String getHash() {+ return hash;+ }++ @Nullable+ public PersonIdent getCommitter() {+ return committer;+ }++ @Nullable+ public PersonIdent getAuthor() {+ return author;+ }++ public static class Builder {++ public String hash;++ public String authorName;++ public String authorEmail;++ public Date authorDate;++ public String committerName;++ public String committerEmail;++ public Date committerDate;++ public List<String> parentHashes = new ArrayList<>();++ public List<String> changedFiles = new ArrayList<>();++ public LogCommit build() {+ PersonIdent committer;+ if (StringUtils.isNotBlank(committerName) && StringUtils.isNotBlank(committerEmail)+ && committerDate != null) {+ committer = GitUtils.newPersonIdent(committerName, committerEmail, committerDate);+ } else {+ committer = null;+ }++ PersonIdent author;+ if (StringUtils.isNotBlank(authorName) && StringUtils.isNotBlank(authorEmail)+ && authorDate != null) {+ author = GitUtils.newPersonIdent(authorName, authorEmail, authorDate);+ } else {+ author = null;+ }++ return new LogCommit(hash, committer, author, parentHashes, changedFiles);+ }+ }++}diff --git a/core/src/main/java/com/gitplex/server/manager/impl/DefaultCommitInfoManager.java b/core/src/main/java/com/gitplex/server/manager/impl/DefaultCommitInfoManager.javaindex 91b7e3680c..58bb6d0497 100644--- a/core/src/main/java/com/gitplex/server/manager/impl/DefaultCommitInfoManager.java+++ b/core/src/main/java/com/gitplex/server/manager/impl/DefaultCommitInfoManager.java@@ -12,10 +12,14 @@ import java.util.HashMap;import java.util.HashSet;import java.util.List;import java.util.Map;+import java.util.Optional;import java.util.Set;import java.util.Stack;import java.util.concurrent.Callable;import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.SynchronousQueue;+import java.util.concurrent.atomic.AtomicReference;import javax.annotation.Nullable;import javax.inject.Inject;@@ -28,16 +32,15 @@ import org.eclipse.jgit.lib.Ref;import org.eclipse.jgit.revwalk.RevCommit;import org.eclipse.jgit.revwalk.RevObject;import org.eclipse.jgit.revwalk.RevWalk;-import org.eclipse.jgit.treewalk.TreeWalk;-import org.eclipse.jgit.treewalk.filter.TreeFilter;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;import com.gitplex.launcher.loader.Listen;import com.gitplex.server.event.RefUpdated;import com.gitplex.server.event.lifecycle.SystemStarted;import com.gitplex.server.git.GitUtils;import com.gitplex.server.git.NameAndEmail;+import com.gitplex.server.git.command.LogCommand;+import com.gitplex.server.git.command.LogCommit;+import com.gitplex.server.git.command.RevListCommand;import com.gitplex.server.manager.BatchWorkManager;import com.gitplex.server.manager.CommitInfoManager;import com.gitplex.server.manager.ProjectManager;@@ -56,6 +59,7 @@ import com.gitplex.server.util.facade.ProjectFacade;import com.gitplex.server.util.facade.UserFacade;import com.google.common.base.Charsets;import com.google.common.base.Splitter;+import com.google.common.base.Throwables;import jetbrains.exodus.ArrayByteIterable;import jetbrains.exodus.ByteIterable;@@ -70,12 +74,14 @@ import jetbrains.exodus.env.TransactionalExecutable;@Singletonpublic class DefaultCommitInfoManager extends AbstractEnvironmentManager implements CommitInfoManager {- private static final Logger logger = LoggerFactory.getLogger(DefaultCommitInfoManager.class);-private static final int INFO_VERSION = 1;private static final long LOG_FILE_SIZE = 256*1024;+ private static final int COLLECT_BATCH_SIZE = 50000;++ private static final int MAX_COLLECTING_FILES = 50000;+private static final String INFO_DIR = "commit";private static final String DEFAULT_STORE = "default";@@ -100,6 +106,8 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager implemeprivate final BatchWorkManager batchWorkManager;+ private final ExecutorService executorService;+private final ProjectManager projectManager;private final UnitOfWork unitOfWork;@@ -116,187 +124,251 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme@Injectpublic DefaultCommitInfoManager(ProjectManager projectManager, StorageManager storageManager,- BatchWorkManager batchWorkManager, UnitOfWork unitOfWork, Dao dao) {+ BatchWorkManager batchWorkManager, UnitOfWork unitOfWork, Dao dao, ExecutorService executorService) {this.projectManager = projectManager;this.storageManager = storageManager;this.batchWorkManager = batchWorkManager;this.unitOfWork = unitOfWork;this.dao = dao;+ this.executorService = executorService;}- private void doCollect(Project project, List<RevCommit> commits) {- logger.debug("Collecting commits info (project: {}, commits: {})...", project, commits.size());-+ private void doCollect(Project project, ObjectId commitId, boolean divide) {Environment env = getEnv(project.getId().toString());Store defaultStore = getStore(env, DEFAULT_STORE);- Store commitsStore = getStore(env, COMMITS_STORE);- Store contributionsStore = getStore(env, CONTRIBUTIONS_STORE);- commits.sort(Comparator.comparing(RevCommit::getCommitTime));-- env.executeInTransaction(new TransactionalExecutable() {-- @SuppressWarnings("unchecked")- @Override- public void execute(Transaction txn) {- byte[] bytes = getBytes(defaultStore.get(txn, LAST_COMMIT_KEY));- ObjectId lastCommitId;- if (bytes != null)- lastCommitId = ObjectId.fromRaw(bytes);- else- lastCommitId = null;-- bytes = getBytes(defaultStore.get(txn, COMMIT_COUNT_KEY));- int prevCommitCount;- if (bytes != null)- prevCommitCount = (int) SerializationUtils.deserialize(bytes);- else- prevCommitCount = 0;- int newCommitCount = prevCommitCount;-- Set<NameAndEmail> prevAuthors;- bytes = getBytes(defaultStore.get(txn, AUTHORS_KEY));- if (bytes != null)- prevAuthors = (Set<NameAndEmail>) SerializationUtils.deserialize(bytes);- else- prevAuthors = new HashSet<>();- Set<NameAndEmail> authors = new HashSet<>(prevAuthors);+ if (divide) {+ List<ObjectId> intermediateCommitIds = new ArrayList<>();+ List<String> revisions = new ArrayList<>();+ revisions.add(commitId.name());++ ObjectId lastCommitId = env.computeInTransaction(new TransactionalComputable<ObjectId>() {++ @Override+ public ObjectId compute(Transaction txn) {+ byte[] bytes = getBytes(defaultStore.get(txn, LAST_COMMIT_KEY));+ return bytes!=null? ObjectId.fromRaw(bytes): null;+ }- Set<NameAndEmail> prevCommitters;- bytes = getBytes(defaultStore.get(txn, COMMITTERS_KEY));- if (bytes != null)- prevCommitters = (Set<NameAndEmail>) SerializationUtils.deserialize(bytes);- else- prevCommitters = new HashSet<>();- Set<NameAndEmail> committers = new HashSet<>(prevCommitters);-- Set<String> prevFiles;- bytes = getBytes(defaultStore.get(txn, FILES_KEY));- if (bytes != null)- prevFiles = (Set<String>) SerializationUtils.deserialize(bytes);- else- prevFiles = new HashSet<>();- Set<String> files = new HashSet<>(prevFiles);+ });+ if (lastCommitId != null) {+ try (RevWalk revWalk = new RevWalk(project.getRepository())) {+ if (GitUtils.parseCommit(revWalk, lastCommitId) != null)+ revisions.add("^" + lastCommitId.name());+ }+ }++ int count = 0;+ for (String commitHash: new RevListCommand(project.getGitDir()).revisions(revisions).call()) {+ count++;+ if (count > COLLECT_BATCH_SIZE) {+ intermediateCommitIds.add(ObjectId.fromString(commitHash));+ count = 0;+ }+ }++ for (int i=intermediateCommitIds.size()-1; i>=0; i--) {+ doCollect(project, intermediateCommitIds.get(i), false);+ }++ doCollect(project, commitId, false);+ } else {+ Store commitsStore = getStore(env, COMMITS_STORE);+ Store contributionsStore = getStore(env, CONTRIBUTIONS_STORE);++ env.executeInTransaction(new TransactionalExecutable() {- try (RevWalk revWalk = new RevWalk(project.getRepository());) {- revWalk.markStart(commits);- if (lastCommitId != null) {- RevCommit lastCommit = GitUtils.parseCommit(revWalk, lastCommitId);- if (lastCommit != null) {- revWalk.markUninteresting(lastCommit);- }- }+ @SuppressWarnings("unchecked")+ @Override+ public void execute(Transaction txn) {+ try {+ byte[] bytes = getBytes(defaultStore.get(txn, LAST_COMMIT_KEY));- RevCommit commit = revWalk.next();- while (commit != null) {- byte[] keyBytes = new byte[20];- commit.copyRawTo(keyBytes, 0);- ByteIterable key = new ArrayByteIterable(keyBytes);- byte[] valueBytes = getBytes(commitsStore.get(txn, key));+ ObjectId lastCommitId;+ if (bytes != null)+ lastCommitId = ObjectId.fromRaw(bytes);+ else+ lastCommitId = null;- if (valueBytes == null || valueBytes.length%2 == 0) {- /*- * Length of stored bytes of a commit is either 20*nChild- * (20 is length of ObjectId), or 1+20*nChild, as we need- * an extra leading byte to differentiate commits being- * processed and commits with child information attached- * but not processed.- */- byte[] newValueBytes;- if (valueBytes == null) {- newValueBytes = new byte[1];- } else {- newValueBytes = new byte[valueBytes.length+1];- System.arraycopy(valueBytes, 0, newValueBytes, 1, valueBytes.length);+ bytes = getBytes(defaultStore.get(txn, COMMIT_COUNT_KEY));+ int prevCommitCount;+ if (bytes != null)+ prevCommitCount = (int) SerializationUtils.deserialize(bytes);+ else+ prevCommitCount = 0;+ int newCommitCount = prevCommitCount;++ Set<NameAndEmail> prevAuthors;+ bytes = getBytes(defaultStore.get(txn, AUTHORS_KEY));+ if (bytes != null)+ prevAuthors = (Set<NameAndEmail>) SerializationUtils.deserialize(bytes);+ else+ prevAuthors = new HashSet<>();+ Set<NameAndEmail> authors = new HashSet<>(prevAuthors);++ Set<NameAndEmail> prevCommitters;+ bytes = getBytes(defaultStore.get(txn, COMMITTERS_KEY));+ if (bytes != null)+ prevCommitters = (Set<NameAndEmail>) SerializationUtils.deserialize(bytes);+ else+ prevCommitters = new HashSet<>();+ Set<NameAndEmail> committers = new HashSet<>(prevCommitters);++ Map<String, Long> files;+ bytes = getBytes(defaultStore.get(txn, FILES_KEY));+ if (bytes != null)+ files = (Map<String, Long>) SerializationUtils.deserialize(bytes);+ else+ files = new HashMap<>();++ /*+ * Use a synchronous queue to achieve below purpose:+ * 1. Add commit to Xodus transactional store in the same thread opening the transaction+ * as this is required by Xodus+ * 2. Do not pile up commits to use minimal memory+ */+ SynchronousQueue<Optional<LogCommit>> queue = new SynchronousQueue<>();+ AtomicReference<Exception> logException = new AtomicReference<>(null);+ executorService.execute(new Runnable() {++ @Override+ public void run() {+ try {+ List<String> revisions = new ArrayList<>();+ revisions.add(commitId.name());+ if (lastCommitId != null)+ revisions.add("^" + lastCommitId.name());++ new LogCommand(project.getGitDir()) {++ @Override+ protected void consume(LogCommit commit) {+ try {+ queue.put(Optional.of(commit));+ } catch (InterruptedException e) {+ }+ }++ }.revisions(revisions).call();++ } catch (Exception e) {+ logException.set(e);+ } finally {+ try {+ queue.put(Optional.empty());+ } catch (InterruptedException e) {+ }+ }}- commitsStore.put(txn, key, new ArrayByteIterable(newValueBytes));- newCommitCount++;+ });++ Optional<LogCommit> commitOptional = queue.take();+ while (commitOptional.isPresent()) {+ LogCommit commit = commitOptional.get();- for (RevCommit parent: commit.getParents()) {- keyBytes = new byte[20];- parent.copyRawTo(keyBytes, 0);- key = new ArrayByteIterable(keyBytes);- valueBytes = getBytes(commitsStore.get(txn, key));- if (valueBytes != null) {- newValueBytes = new byte[valueBytes.length+20];- System.arraycopy(valueBytes, 0, newValueBytes, 0, valueBytes.length);+ byte[] keyBytes = new byte[20];+ ObjectId.fromString(commit.getHash()).copyRawTo(keyBytes, 0);+ ByteIterable key = new ArrayByteIterable(keyBytes);+ byte[] valueBytes = getBytes(commitsStore.get(txn, key));++ if (valueBytes == null || valueBytes.length % 20 == 0) {+ /*+ * Length of stored bytes of a commit is either 20*nChild+ * (20 is length of ObjectId), or 1+20*nChild, as we need+ * an extra leading byte to differentiate commits being+ * processed and commits with child information attached+ * but not processed.+ */+ byte[] newValueBytes;+ if (valueBytes == null) {+ newValueBytes = new byte[1];} else {- newValueBytes = new byte[20];+ newValueBytes = new byte[valueBytes.length+1];+ System.arraycopy(valueBytes, 0, newValueBytes, 1, valueBytes.length);}- commit.copyRawTo(newValueBytes, newValueBytes.length-20);commitsStore.put(txn, key, new ArrayByteIterable(newValueBytes));- }-- if (StringUtils.isNotBlank(commit.getAuthorIdent().getName())- || StringUtils.isNotBlank(commit.getAuthorIdent().getEmailAddress())) {- authors.add(new NameAndEmail(commit.getAuthorIdent()));- }++ newCommitCount++;++ for (String parentHash: commit.getParentHashes()) {+ keyBytes = new byte[20];+ ObjectId.fromString(parentHash).copyRawTo(keyBytes, 0);+ key = new ArrayByteIterable(keyBytes);+ valueBytes = getBytes(commitsStore.get(txn, key));+ if (valueBytes != null) {+ newValueBytes = new byte[valueBytes.length+20];+ System.arraycopy(valueBytes, 0, newValueBytes, 0, valueBytes.length);+ } else {+ newValueBytes = new byte[20];+ }+ ObjectId.fromString(commit.getHash()).copyRawTo(newValueBytes, newValueBytes.length-20);+ commitsStore.put(txn, key, new ArrayByteIterable(newValueBytes));+ }++ if (commit.getCommitter() != null) {+ committers.add(new NameAndEmail(commit.getCommitter()));+ for (String file: commit.getChangedFiles())+ files.put(file, commit.getCommitter().getWhen().getTime());+ }- if (StringUtils.isNotBlank(commit.getCommitterIdent().getName())- || StringUtils.isNotBlank(commit.getCommitterIdent().getEmailAddress())) {- committers.add(new NameAndEmail(commit.getCommitterIdent()));- }-- String authorEmail = commit.getAuthorIdent().getEmailAddress();- if (StringUtils.isNotBlank(authorEmail) && commit.getParentCount() == 1) {- try (TreeWalk treeWalk = new TreeWalk(project.getRepository())) {- treeWalk.setFilter(TreeFilter.ANY_DIFF);- treeWalk.addTree(commit.getTree());- treeWalk.addTree(commit.getParent(0).getTree());- treeWalk.setRecursive(true);- while (treeWalk.next()) {- String path = treeWalk.getPathString();- files.add(path);-- updateContribution(txn, contributionsStore, authorEmail, path);+ if (commit.getAuthor() != null) {+ authors.add(new NameAndEmail(commit.getAuthor()));+ for (String path: commit.getChangedFiles()) {+ updateContribution(txn, contributionsStore, commit.getAuthor().getEmailAddress(), path);while (path.contains("/")) {path = StringUtils.substringBeforeLast(path, "/");- updateContribution(txn, contributionsStore, authorEmail, path);+ updateContribution(txn, contributionsStore,+ commit.getAuthor().getEmailAddress(), path);}- updateContribution(txn, contributionsStore, authorEmail, "");+ updateContribution(txn, contributionsStore,+ commit.getAuthor().getEmailAddress(), "");}}- }- }- commit = revWalk.next();+ }+ commitOptional = queue.take();+ }+ if (logException.get() != null)+ throw logException.get();++ if (newCommitCount != prevCommitCount) {+ bytes = SerializationUtils.serialize(newCommitCount);+ defaultStore.put(txn, COMMIT_COUNT_KEY, new ArrayByteIterable(bytes));+ commitCountCache.put(project.getId(), newCommitCount);+ }++ if (!authors.equals(prevAuthors)) {+ bytes = SerializationUtils.serialize((Serializable) authors);+ defaultStore.put(txn, AUTHORS_KEY, new ArrayByteIterable(bytes));+ authorsCache.remove(project.getId());+ }++ if (!committers.equals(prevCommitters)) {+ bytes = SerializationUtils.serialize((Serializable) committers);+ defaultStore.put(txn, COMMITTERS_KEY, new ArrayByteIterable(bytes));+ committersCache.remove(project.getId());+ }++ if (files.size() > MAX_COLLECTING_FILES) {+ List<String> fileList = new ArrayList<>(files.keySet());+ fileList.sort((file1, file2)->files.get(file1).compareTo(files.get(file2)));+ for (int i=0; i<fileList.size() - MAX_COLLECTING_FILES; i++)+ files.remove(fileList.get(i));+ }+ bytes = SerializationUtils.serialize((Serializable) files);+ defaultStore.put(txn, FILES_KEY, new ArrayByteIterable(bytes));+ filesCache.remove(project.getId());++ bytes = new byte[20];+ commitId.copyRawTo(bytes, 0);+ defaultStore.put(txn, LAST_COMMIT_KEY, new ArrayByteIterable(bytes));+ } catch (Exception e) {+ Throwables.propagate(e);}-- } catch (IOException e) {- throw new RuntimeException(e);- }-- if (newCommitCount != prevCommitCount) {- bytes = SerializationUtils.serialize(newCommitCount);- defaultStore.put(txn, COMMIT_COUNT_KEY, new ArrayByteIterable(bytes));- commitCountCache.put(project.getId(), newCommitCount);- }-- if (!authors.equals(prevAuthors)) {- bytes = SerializationUtils.serialize((Serializable) authors);- defaultStore.put(txn, AUTHORS_KEY, new ArrayByteIterable(bytes));- authorsCache.remove(project.getId());- }-- if (!committers.equals(prevCommitters)) {- bytes = SerializationUtils.serialize((Serializable) committers);- defaultStore.put(txn, COMMITTERS_KEY, new ArrayByteIterable(bytes));- committersCache.remove(project.getId());- }-- if (!files.equals(prevFiles)) {- bytes = SerializationUtils.serialize((Serializable) files);- defaultStore.put(txn, FILES_KEY, new ArrayByteIterable(bytes));- filesCache.remove(project.getId());}- bytes = new byte[20];- commits.get(commits.size()-1).copyRawTo(bytes, 0);- defaultStore.put(txn, LAST_COMMIT_KEY, new ArrayByteIterable(bytes));- }-- });- logger.debug("Commits info collected (project: {}, commits: {})", project, commits.size());+ });+ }}private void updateContribution(Transaction txn, Store contributionsStore, String email, String path) {@@ -385,7 +457,8 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager implemepublic List<String> compute(Transaction txn) {byte[] bytes = getBytes(store.get(txn, FILES_KEY));if (bytes != null) {- List<String> files = new ArrayList<>((Set<String>)SerializationUtils.deserialize(bytes));+ List<String> files = new ArrayList<>(+ ((Map<String, Long>)SerializationUtils.deserialize(bytes)).keySet());Map<String, List<String>> segmentsMap = new HashMap<>();Splitter splitter = Splitter.on("/");for (String file: files) {@@ -457,7 +530,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager implemecurrent.copyRawTo(keyBytes, 0);byte[] valueBytes = getBytes(store.get(txn, new ArrayByteIterable(keyBytes)));if (valueBytes != null) {- if (valueBytes.length % 2 == 0) {+ if (valueBytes.length % 20 == 0) {for (int i=0; i<valueBytes.length/20; i++) {ObjectId child = ObjectId.fromRaw(valueBytes, i*20);if (!descendants.contains(child)) {@@ -502,7 +575,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager implemeparent.copyRawTo(keyBytes, 0);byte[] valueBytes = getBytes(store.get(txn, new ArrayByteIterable(keyBytes)));if (valueBytes != null) {- if (valueBytes.length % 2 == 0) {+ if (valueBytes.length % 20 == 0) {for (int i=0; i<valueBytes.length/20; i++) {children.add(ObjectId.fromRaw(valueBytes, i*20));}@@ -551,14 +624,22 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme@Overridepublic Void call() throws Exception {Project project = projectManager.load(projectId);- List<RevCommit> commits = new ArrayList<>();- try (RevWalk revWalk = new RevWalk(project.getRepository())) {- for (Prioritized work: works) {- CollectingWork collectingWork = (CollectingWork) work;- commits.add(revWalk.parseCommit(collectingWork.getCommitId()));+ List<ObjectId> commitIds = new ArrayList<>();+ if (works.size() > 1) {+ try (RevWalk revWalk = new RevWalk(project.getRepository())) {+ List<RevCommit> commits = new ArrayList<>();+ for (Object work: works)+ commits.add(revWalk.parseCommit(((CollectingWork)work).getCommitId()));+ commits.sort(Comparator.comparing(RevCommit::getCommitTime));+ for (RevCommit commit: commits)+ commitIds.add(commit.copy());}+ } else {+ CollectingWork collectingWork = (CollectingWork) works.iterator().next();+ commitIds.add(collectingWork.getCommitId());}- doCollect(project, commits);+ for (ObjectId commitId: commitIds)+ doCollect(project, commitId, true);return null;}@@ -589,7 +670,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager implemecommit.copyRawTo(keyBytes, 0);ByteIterable key = new ArrayByteIterable(keyBytes);byte[] valueBytes = getBytes(commitsStore.get(txn, key));- return valueBytes != null && valueBytes.length%2 != 0;+ return valueBytes != null && valueBytes.length % 20 != 0;}});@@ -603,7 +684,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager implemecommits.sort(Comparator.comparing(RevCommit::getCommitTime));for (RevCommit commit: commits) {- CollectingWork work = new CollectingWork(PRIORITY, commit.getId());+ CollectingWork work = new CollectingWork(PRIORITY, commit.copy());batchWorkManager.submit(getBatchWorker(project), work);}}@@ -620,11 +701,8 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme@Listenpublic void on(RefUpdated event) {if (!event.getNewObjectId().equals(ObjectId.zeroId())) {- RevCommit commit = event.getProject().getRevCommit(event.getNewObjectId(), false);- if (commit != null) {- CollectingWork work = new CollectingWork(PRIORITY, commit.getId());- batchWorkManager.submit(getBatchWorker(event.getProject()), work);- }+ CollectingWork work = new CollectingWork(PRIORITY, event.getNewObjectId());+ batchWorkManager.submit(getBatchWorker(event.getProject()), work);}}diff --git a/web/src/main/java/com/gitplex/server/web/component/diff/blob/text/TextDiffPanel.java b/web/src/main/java/com/gitplex/server/web/component/diff/blob/text/TextDiffPanel.javaindex 191fbea532..115dc0671f 100644--- a/web/src/main/java/com/gitplex/server/web/component/diff/blob/text/TextDiffPanel.java+++ b/web/src/main/java/com/gitplex/server/web/component/diff/blob/text/TextDiffPanel.java@@ -45,7 +45,7 @@ import com.gitplex.server.GitPlex;import com.gitplex.server.git.Blame;import com.gitplex.server.git.BlobChange;import com.gitplex.server.git.BlobIdent;-import com.gitplex.server.git.Commit;+import com.gitplex.server.git.BlameCommit;import com.gitplex.server.git.GitUtils;import com.gitplex.server.git.command.BlameCommand;import com.gitplex.server.manager.CodeCommentManager;@@ -705,7 +705,7 @@ public class TextDiffPanel extends Panel implements SourceAware {}private void appendBlame(StringBuilder builder, int oldLineNo, int newLineNo) {- Commit commit;+ BlameCommit commit;if (newLineNo != -1)commit = Preconditions.checkNotNull(blameInfo.newBlame.get(newLineNo));else@@ -1208,9 +1208,9 @@ public class TextDiffPanel extends Panel implements SourceAware {private static class BlameInfo implements Serializable {- Map<Integer, Commit> oldBlame = new HashMap<>();+ Map<Integer, BlameCommit> oldBlame = new HashMap<>();- Map<Integer, Commit> newBlame = new HashMap<>();+ Map<Integer, BlameCommit> newBlame = new HashMap<>();String lastCommitHash;diff --git a/web/src/main/java/com/gitplex/server/web/page/base/base.js b/web/src/main/java/com/gitplex/server/web/page/base/base.jsindex 436ad14376..fcc0184111 100644--- a/web/src/main/java/com/gitplex/server/web/page/base/base.js+++ b/web/src/main/java/com/gitplex/server/web/page/base/base.js@@ -248,23 +248,30 @@ gitplex.server = {e.stopPropagation();});+ var ajaxCalls = 0;Wicket.Event.subscribe('/ajax/call/beforeSend', function() {- var $ajaxLoadingIndicator = $("#ajax-loading-indicator");- if ($ajaxLoadingIndicator[0].timer)- clearTimeout($ajaxLoadingIndicator[0].timer);- $ajaxLoadingIndicator[0].timer = setTimeout(function() {- if (!$ajaxLoadingIndicator.is(":visible") && $(".ajax-indicator:visible").length == 0)- $ajaxLoadingIndicator.show();- }, 2000);+ if (ajaxCalls == 0) {+ var $ajaxLoadingIndicator = $("#ajax-loading-indicator");+ if ($ajaxLoadingIndicator[0].timer)+ clearTimeout($ajaxLoadingIndicator[0].timer);+ $ajaxLoadingIndicator[0].timer = setTimeout(function() {+ if (!$ajaxLoadingIndicator.is(":visible") && $(".ajax-indicator:visible").length == 0)+ $ajaxLoadingIndicator.show();+ }, 2000);+ }+ ajaxCalls++;});- Wicket.Event.subscribe('/ajax/call/complete', function() {- var $ajaxLoadingIndicator = $("#ajax-loading-indicator");- if ($ajaxLoadingIndicator[0].timer) {- clearTimeout($ajaxLoadingIndicator[0].timer);- $ajaxLoadingIndicator[0].timer = null;+ Wicket.Event.subscribe('/ajax/call/done', function() {+ ajaxCalls--;+ if (ajaxCalls == 0) {+ var $ajaxLoadingIndicator = $("#ajax-loading-indicator");+ if ($ajaxLoadingIndicator[0].timer) {+ clearTimeout($ajaxLoadingIndicator[0].timer);+ $ajaxLoadingIndicator[0].timer = null;+ }+ $ajaxLoadingIndicator.hide();}- $ajaxLoadingIndicator.hide();});},diff --git a/web/src/main/java/com/gitplex/server/web/page/project/commit/ProjectCommitsPage.java b/web/src/main/java/com/gitplex/server/web/page/project/commit/ProjectCommitsPage.javaindex d5d4b1c279..a6736c251c 100644--- a/web/src/main/java/com/gitplex/server/web/page/project/commit/ProjectCommitsPage.java+++ b/web/src/main/java/com/gitplex/server/web/page/project/commit/ProjectCommitsPage.java@@ -112,7 +112,6 @@ public class ProjectCommitsPage extends ProjectPage {@Overrideprotected Commits load() {Commits commits = new Commits();-List<String> commitHashes;try {RevListCommand command = new RevListCommand(getProject().getGitDir());