Skip to content

The Unified and Autonomous Development Platform

Some performance improvements and bug fixes

robin shen2ffa9028c9ab
diff --git a/core/src/main/java/com/gitplex/server/git/Blame.java b/core/src/main/java/com/gitplex/server/git/Blame.java
index d4a421c1c4..92f889f137 100644
--- a/core/src/main/java/com/gitplex/server/git/Blame.java
+++ b/core/src/main/java/com/gitplex/server/git/Blame.java
@@ -8,16 +8,16 @@ import com.gitplex.jsymbol.Range;
public class Blame {
- private final Commit commit;
+ private final BlameCommit commit;
private final List<Range> ranges;
- public Blame(Commit commit, List<Range> ranges) {
+ public Blame(BlameCommit commit, List<Range> ranges) {
this.commit = commit;
this.ranges = ranges;
}
- public Commit getCommit() {
+ public BlameCommit getCommit() {
return commit;
}
diff --git a/core/src/main/java/com/gitplex/server/git/Commit.java b/core/src/main/java/com/gitplex/server/git/BlameCommit.java
similarity index 83%
rename from core/src/main/java/com/gitplex/server/git/Commit.java
rename to core/src/main/java/com/gitplex/server/git/BlameCommit.java
index b24bd1330c..f569288c83 100644
--- a/core/src/main/java/com/gitplex/server/git/Commit.java
+++ b/core/src/main/java/com/gitplex/server/git/BlameCommit.java
@@ -9,7 +9,7 @@ import org.eclipse.jgit.lib.PersonIdent;
import com.google.common.base.Objects;
-public class Commit implements Serializable {
+public class BlameCommit implements Serializable {
private static final long serialVersionUID = 1L;
@@ -21,14 +21,14 @@ public class Commit implements Serializable {
private final PersonIdent author;
- public Commit(String hash, PersonIdent committer, PersonIdent author, String subject) {
+ public BlameCommit(String hash, PersonIdent committer, PersonIdent author, String subject) {
this.hash = checkNotNull(hash, "hash");
this.committer = checkNotNull(committer, "committer");
this.author = checkNotNull(author, "author");
this.subject = checkNotNull(subject, "subject");
}
- public Commit(Commit commit) {
+ public BlameCommit(BlameCommit commit) {
this(commit.getHash(), commit.getCommitter(), commit.getAuthor(), commit.getSubject());
}
@@ -56,9 +56,9 @@ public class Commit implements Serializable {
@Override
public boolean equals(Object other) {
if (this == other) return true;
- if (!(other instanceof Commit)) return false;
+ if (!(other instanceof BlameCommit)) return false;
- Commit rhs = (Commit) other;
+ BlameCommit rhs = (BlameCommit) other;
return Objects.equal(hash, rhs.hash);
}
@@ -90,10 +90,10 @@ public class Commit implements Serializable {
public String subject;
- public Commit build() {
+ public BlameCommit build() {
PersonIdent committer = GitUtils.newPersonIdent(committerName, committerEmail, committerDate);
PersonIdent author = GitUtils.newPersonIdent(authorName, authorEmail, authorDate);
- return new Commit(hash, committer, author, subject.trim());
+ return new BlameCommit(hash, committer, author, subject.trim());
}
}
}
diff --git a/core/src/main/java/com/gitplex/server/git/command/BlameCommand.java b/core/src/main/java/com/gitplex/server/git/command/BlameCommand.java
index 6d222dd546..182416bdd8 100644
--- a/core/src/main/java/com/gitplex/server/git/command/BlameCommand.java
+++ b/core/src/main/java/com/gitplex/server/git/command/BlameCommand.java
@@ -17,7 +17,7 @@ import org.slf4j.LoggerFactory;
import com.gitplex.jsymbol.Range;
import com.gitplex.server.git.Blame;
-import com.gitplex.server.git.Commit;
+import com.gitplex.server.git.BlameCommit;
import com.gitplex.server.git.GitUtils;
import com.gitplex.server.util.execution.Commandline;
import com.gitplex.server.util.execution.ExecuteResult;
@@ -71,10 +71,10 @@ public class BlameCommand extends GitCommand<Map<String, Blame>> {
Commandline cmd = buildCmd();
final Map<String, Blame> blames = new HashMap<>();
- final Map<String, Commit> commitMap = new HashMap<>();
+ final Map<String, BlameCommit> commitMap = new HashMap<>();
- final AtomicReference<Commit> commitRef = new AtomicReference<>(null);
- final BriefCommitBuilder commitBuilder = new BriefCommitBuilder();
+ final AtomicReference<BlameCommit> commitRef = new AtomicReference<>(null);
+ final CommitBuilder commitBuilder = new CommitBuilder();
final AtomicBoolean endOfFile = new AtomicBoolean(false);
final AtomicInteger beginLine = new AtomicInteger(0);
@@ -93,7 +93,7 @@ public class BlameCommand extends GitCommand<Map<String, Blame>> {
commitBuilder.hash = null;
} else if (commitBuilder.hash == null) {
commitBuilder.hash = StringUtils.substringBefore(line, " ");
- Commit commit = commitRef.get();
+ BlameCommit commit = commitRef.get();
if (commit != null && !commitBuilder.hash.equals(commit.getHash())) {
Blame blame = blames.get(commit.getHash());
if (blame == null) {
@@ -142,7 +142,7 @@ public class BlameCommand extends GitCommand<Map<String, Blame>> {
result.checkReturnCode();
if (endLine.get() > beginLine.get()) {
- Commit commit = commitRef.get();
+ BlameCommit commit = commitRef.get();
Blame blame = blames.get(commit.getHash());
if (blame == null) {
blame = new Blame(commit, new ArrayList<Range>());
@@ -157,7 +157,7 @@ public class BlameCommand extends GitCommand<Map<String, Blame>> {
return blames;
}
- private static class BriefCommitBuilder {
+ private static class CommitBuilder {
private Date committerDate;
@@ -175,8 +175,8 @@ public class BlameCommand extends GitCommand<Map<String, Blame>> {
private String summary;
- private Commit build() {
- return new Commit(
+ private BlameCommit build() {
+ return new BlameCommit(
hash,
GitUtils.newPersonIdent(committer, committerEmail, committerDate),
GitUtils.newPersonIdent(author, authorEmail, authorDate),
diff --git a/core/src/main/java/com/gitplex/server/git/command/LogCommand.java b/core/src/main/java/com/gitplex/server/git/command/LogCommand.java
new file mode 100644
index 0000000000..7ea198bb5c
--- /dev/null
+++ b/core/src/main/java/com/gitplex/server/git/command/LogCommand.java
@@ -0,0 +1,117 @@
+package com.gitplex.server.git.command;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.commons.lang3.StringUtils;
+import org.eclipse.jgit.util.QuotedString;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.gitplex.server.git.GitUtils;
+import com.gitplex.server.util.execution.Commandline;
+import com.gitplex.server.util.execution.LineConsumer;
+
+public abstract class LogCommand extends GitCommand<Void> {
+
+ private static final Logger logger = LoggerFactory.getLogger(LogCommand.class);
+
+ private List<String> revisions = new ArrayList<>();
+
+ public LogCommand(File gitDir) {
+ super(gitDir);
+ }
+
+ public List<String> revisions() {
+ return revisions;
+ }
+
+ public LogCommand revisions(List<String> revisions) {
+ this.revisions = revisions;
+ return this;
+ }
+
+ @Override
+ public Void call() {
+ Commandline cmd = cmd();
+ cmd.addArgs("log",
+ "--format=*** commit_begin ***%nhash:%H%nauthor:%aN%nauthorEmail:%aE%n"
+ + "committerEmail:%cE%ncommitter:%cN%nparents:%P%ncommitterDate:%cd %n"
+ + "authorDate:%ad %n",
+ "--date=raw");
+
+ if (!revisions.isEmpty()) {
+ for (String revision: revisions)
+ cmd.addArgs(revision);
+ } else {
+ cmd.addArgs("--branches");
+ }
+
+ cmd.addArgs("--name-only", "--no-renames");
+
+ LogCommit.Builder commitBuilder = new LogCommit.Builder();
+
+ AtomicBoolean changedFilesBlock = new AtomicBoolean();
+
+ cmd.execute(new LineConsumer() {
+
+ @Override
+ public void consume(String line) {
+ if (line.equals("*** commit_begin ***")) {
+ if (commitBuilder.hash!= null)
+ LogCommand.this.consume(commitBuilder.build());
+ commitBuilder.hash = null;
+ commitBuilder.committerName = null;
+ commitBuilder.committerEmail = null;
+ commitBuilder.committerDate = null;
+ commitBuilder.authorName = null;
+ commitBuilder.authorEmail = null;
+ commitBuilder.authorDate = null;
+ commitBuilder.parentHashes = new ArrayList<>();
+ commitBuilder.changedFiles = new ArrayList<>();
+ changedFilesBlock.set(false);
+ } else if (changedFilesBlock.get()) {
+ if (line.trim().length() != 0 && commitBuilder.changedFiles != null)
+ commitBuilder.changedFiles.add(QuotedString.GIT_PATH.dequote(line));
+ } else if (line.startsWith("hash:")) {
+ commitBuilder.hash = line.substring("hash:".length());
+ } else if (line.startsWith("author:")) {
+ commitBuilder.authorName = line.substring("author:".length());
+ } else if (line.startsWith("committer:")) {
+ commitBuilder.committerName = line.substring("committer:".length());
+ } else if (line.startsWith("authorEmail:")) {
+ commitBuilder.authorEmail = line.substring("authorEmail:".length());
+ } else if (line.startsWith("committerEmail:")) {
+ commitBuilder.committerEmail = line.substring("committerEmail:".length());
+ } else if (line.startsWith("parents:")) {
+ for (String each: StringUtils.split(line.substring("parents:".length()), " "))
+ commitBuilder.parentHashes.add(each);
+ } else if (line.startsWith("committerDate:")) {
+ commitBuilder.committerDate = GitUtils.parseRawDate(line.substring("committerDate:".length()).trim());
+ } else if (line.startsWith("authorDate:")) {
+ commitBuilder.authorDate = GitUtils.parseRawDate(line.substring("authorDate:".length()).trim());
+ if (commitBuilder.changedFiles != null)
+ changedFilesBlock.set(true);
+ }
+ }
+
+ }, new LineConsumer() {
+
+ @Override
+ public void consume(String line) {
+ logger.error(line);
+ }
+
+ }).checkReturnCode();
+
+ if (commitBuilder.hash != null)
+ consume(commitBuilder.build());
+
+ return null;
+ }
+
+ protected abstract void consume(LogCommit commit);
+
+}
diff --git a/core/src/main/java/com/gitplex/server/git/command/LogCommit.java b/core/src/main/java/com/gitplex/server/git/command/LogCommit.java
new file mode 100644
index 0000000000..e2fdf4204a
--- /dev/null
+++ b/core/src/main/java/com/gitplex/server/git/command/LogCommit.java
@@ -0,0 +1,101 @@
+package com.gitplex.server.git.command;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+import javax.annotation.Nullable;
+
+import org.apache.commons.lang3.StringUtils;
+import org.eclipse.jgit.lib.PersonIdent;
+
+import com.gitplex.server.git.GitUtils;
+
+public class LogCommit implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private final String hash;
+
+ private final PersonIdent committer;
+
+ private final PersonIdent author;
+
+ private final List<String> parentHashes;
+
+ private final List<String> changedFiles;
+
+ public LogCommit(String hash, @Nullable PersonIdent committer, @Nullable PersonIdent author,
+ List<String> parentHashes, List<String> changedFiles) {
+ this.hash = hash;
+ this.committer = committer;
+ this.author = author;
+ this.parentHashes = parentHashes;
+ this.changedFiles = changedFiles;
+ }
+
+ public List<String> getParentHashes() {
+ return parentHashes;
+ }
+
+ public List<String> getChangedFiles() {
+ return changedFiles;
+ }
+
+ public String getHash() {
+ return hash;
+ }
+
+ @Nullable
+ public PersonIdent getCommitter() {
+ return committer;
+ }
+
+ @Nullable
+ public PersonIdent getAuthor() {
+ return author;
+ }
+
+ public static class Builder {
+
+ public String hash;
+
+ public String authorName;
+
+ public String authorEmail;
+
+ public Date authorDate;
+
+ public String committerName;
+
+ public String committerEmail;
+
+ public Date committerDate;
+
+ public List<String> parentHashes = new ArrayList<>();
+
+ public List<String> changedFiles = new ArrayList<>();
+
+ public LogCommit build() {
+ PersonIdent committer;
+ if (StringUtils.isNotBlank(committerName) && StringUtils.isNotBlank(committerEmail)
+ && committerDate != null) {
+ committer = GitUtils.newPersonIdent(committerName, committerEmail, committerDate);
+ } else {
+ committer = null;
+ }
+
+ PersonIdent author;
+ if (StringUtils.isNotBlank(authorName) && StringUtils.isNotBlank(authorEmail)
+ && authorDate != null) {
+ author = GitUtils.newPersonIdent(authorName, authorEmail, authorDate);
+ } else {
+ author = null;
+ }
+
+ return new LogCommit(hash, committer, author, parentHashes, changedFiles);
+ }
+ }
+
+}
diff --git a/core/src/main/java/com/gitplex/server/manager/impl/DefaultCommitInfoManager.java b/core/src/main/java/com/gitplex/server/manager/impl/DefaultCommitInfoManager.java
index 91b7e3680c..58bb6d0497 100644
--- a/core/src/main/java/com/gitplex/server/manager/impl/DefaultCommitInfoManager.java
+++ b/core/src/main/java/com/gitplex/server/manager/impl/DefaultCommitInfoManager.java
@@ -12,10 +12,14 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import java.util.Stack;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.atomic.AtomicReference;
import javax.annotation.Nullable;
import javax.inject.Inject;
@@ -28,16 +32,15 @@ import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevObject;
import org.eclipse.jgit.revwalk.RevWalk;
-import org.eclipse.jgit.treewalk.TreeWalk;
-import org.eclipse.jgit.treewalk.filter.TreeFilter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import com.gitplex.launcher.loader.Listen;
import com.gitplex.server.event.RefUpdated;
import com.gitplex.server.event.lifecycle.SystemStarted;
import com.gitplex.server.git.GitUtils;
import com.gitplex.server.git.NameAndEmail;
+import com.gitplex.server.git.command.LogCommand;
+import com.gitplex.server.git.command.LogCommit;
+import com.gitplex.server.git.command.RevListCommand;
import com.gitplex.server.manager.BatchWorkManager;
import com.gitplex.server.manager.CommitInfoManager;
import com.gitplex.server.manager.ProjectManager;
@@ -56,6 +59,7 @@ import com.gitplex.server.util.facade.ProjectFacade;
import com.gitplex.server.util.facade.UserFacade;
import com.google.common.base.Charsets;
import com.google.common.base.Splitter;
+import com.google.common.base.Throwables;
import jetbrains.exodus.ArrayByteIterable;
import jetbrains.exodus.ByteIterable;
@@ -70,12 +74,14 @@ import jetbrains.exodus.env.TransactionalExecutable;
@Singleton
public class DefaultCommitInfoManager extends AbstractEnvironmentManager implements CommitInfoManager {
- private static final Logger logger = LoggerFactory.getLogger(DefaultCommitInfoManager.class);
-
private static final int INFO_VERSION = 1;
private static final long LOG_FILE_SIZE = 256*1024;
+ private static final int COLLECT_BATCH_SIZE = 50000;
+
+ private static final int MAX_COLLECTING_FILES = 50000;
+
private static final String INFO_DIR = "commit";
private static final String DEFAULT_STORE = "default";
@@ -100,6 +106,8 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme
private final BatchWorkManager batchWorkManager;
+ private final ExecutorService executorService;
+
private final ProjectManager projectManager;
private final UnitOfWork unitOfWork;
@@ -116,187 +124,251 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme
@Inject
public DefaultCommitInfoManager(ProjectManager projectManager, StorageManager storageManager,
- BatchWorkManager batchWorkManager, UnitOfWork unitOfWork, Dao dao) {
+ BatchWorkManager batchWorkManager, UnitOfWork unitOfWork, Dao dao, ExecutorService executorService) {
this.projectManager = projectManager;
this.storageManager = storageManager;
this.batchWorkManager = batchWorkManager;
this.unitOfWork = unitOfWork;
this.dao = dao;
+ this.executorService = executorService;
}
- private void doCollect(Project project, List<RevCommit> commits) {
- logger.debug("Collecting commits info (project: {}, commits: {})...", project, commits.size());
-
+ private void doCollect(Project project, ObjectId commitId, boolean divide) {
Environment env = getEnv(project.getId().toString());
Store defaultStore = getStore(env, DEFAULT_STORE);
- Store commitsStore = getStore(env, COMMITS_STORE);
- Store contributionsStore = getStore(env, CONTRIBUTIONS_STORE);
- commits.sort(Comparator.comparing(RevCommit::getCommitTime));
-
- env.executeInTransaction(new TransactionalExecutable() {
-
- @SuppressWarnings("unchecked")
- @Override
- public void execute(Transaction txn) {
- byte[] bytes = getBytes(defaultStore.get(txn, LAST_COMMIT_KEY));
- ObjectId lastCommitId;
- if (bytes != null)
- lastCommitId = ObjectId.fromRaw(bytes);
- else
- lastCommitId = null;
-
- bytes = getBytes(defaultStore.get(txn, COMMIT_COUNT_KEY));
- int prevCommitCount;
- if (bytes != null)
- prevCommitCount = (int) SerializationUtils.deserialize(bytes);
- else
- prevCommitCount = 0;
- int newCommitCount = prevCommitCount;
-
- Set<NameAndEmail> prevAuthors;
- bytes = getBytes(defaultStore.get(txn, AUTHORS_KEY));
- if (bytes != null)
- prevAuthors = (Set<NameAndEmail>) SerializationUtils.deserialize(bytes);
- else
- prevAuthors = new HashSet<>();
- Set<NameAndEmail> authors = new HashSet<>(prevAuthors);
+ if (divide) {
+ List<ObjectId> intermediateCommitIds = new ArrayList<>();
+ List<String> revisions = new ArrayList<>();
+ revisions.add(commitId.name());
+
+ ObjectId lastCommitId = env.computeInTransaction(new TransactionalComputable<ObjectId>() {
+
+ @Override
+ public ObjectId compute(Transaction txn) {
+ byte[] bytes = getBytes(defaultStore.get(txn, LAST_COMMIT_KEY));
+ return bytes!=null? ObjectId.fromRaw(bytes): null;
+ }
- Set<NameAndEmail> prevCommitters;
- bytes = getBytes(defaultStore.get(txn, COMMITTERS_KEY));
- if (bytes != null)
- prevCommitters = (Set<NameAndEmail>) SerializationUtils.deserialize(bytes);
- else
- prevCommitters = new HashSet<>();
- Set<NameAndEmail> committers = new HashSet<>(prevCommitters);
-
- Set<String> prevFiles;
- bytes = getBytes(defaultStore.get(txn, FILES_KEY));
- if (bytes != null)
- prevFiles = (Set<String>) SerializationUtils.deserialize(bytes);
- else
- prevFiles = new HashSet<>();
- Set<String> files = new HashSet<>(prevFiles);
+ });
+ if (lastCommitId != null) {
+ try (RevWalk revWalk = new RevWalk(project.getRepository())) {
+ if (GitUtils.parseCommit(revWalk, lastCommitId) != null)
+ revisions.add("^" + lastCommitId.name());
+ }
+ }
+
+ int count = 0;
+ for (String commitHash: new RevListCommand(project.getGitDir()).revisions(revisions).call()) {
+ count++;
+ if (count > COLLECT_BATCH_SIZE) {
+ intermediateCommitIds.add(ObjectId.fromString(commitHash));
+ count = 0;
+ }
+ }
+
+ for (int i=intermediateCommitIds.size()-1; i>=0; i--) {
+ doCollect(project, intermediateCommitIds.get(i), false);
+ }
+
+ doCollect(project, commitId, false);
+ } else {
+ Store commitsStore = getStore(env, COMMITS_STORE);
+ Store contributionsStore = getStore(env, CONTRIBUTIONS_STORE);
+
+ env.executeInTransaction(new TransactionalExecutable() {
- try (RevWalk revWalk = new RevWalk(project.getRepository());) {
- revWalk.markStart(commits);
- if (lastCommitId != null) {
- RevCommit lastCommit = GitUtils.parseCommit(revWalk, lastCommitId);
- if (lastCommit != null) {
- revWalk.markUninteresting(lastCommit);
- }
- }
+ @SuppressWarnings("unchecked")
+ @Override
+ public void execute(Transaction txn) {
+ try {
+ byte[] bytes = getBytes(defaultStore.get(txn, LAST_COMMIT_KEY));
- RevCommit commit = revWalk.next();
- while (commit != null) {
- byte[] keyBytes = new byte[20];
- commit.copyRawTo(keyBytes, 0);
- ByteIterable key = new ArrayByteIterable(keyBytes);
- byte[] valueBytes = getBytes(commitsStore.get(txn, key));
+ ObjectId lastCommitId;
+ if (bytes != null)
+ lastCommitId = ObjectId.fromRaw(bytes);
+ else
+ lastCommitId = null;
- if (valueBytes == null || valueBytes.length%2 == 0) {
- /*
- * Length of stored bytes of a commit is either 20*nChild
- * (20 is length of ObjectId), or 1+20*nChild, as we need
- * an extra leading byte to differentiate commits being
- * processed and commits with child information attached
- * but not processed.
- */
- byte[] newValueBytes;
- if (valueBytes == null) {
- newValueBytes = new byte[1];
- } else {
- newValueBytes = new byte[valueBytes.length+1];
- System.arraycopy(valueBytes, 0, newValueBytes, 1, valueBytes.length);
+ bytes = getBytes(defaultStore.get(txn, COMMIT_COUNT_KEY));
+ int prevCommitCount;
+ if (bytes != null)
+ prevCommitCount = (int) SerializationUtils.deserialize(bytes);
+ else
+ prevCommitCount = 0;
+ int newCommitCount = prevCommitCount;
+
+ Set<NameAndEmail> prevAuthors;
+ bytes = getBytes(defaultStore.get(txn, AUTHORS_KEY));
+ if (bytes != null)
+ prevAuthors = (Set<NameAndEmail>) SerializationUtils.deserialize(bytes);
+ else
+ prevAuthors = new HashSet<>();
+ Set<NameAndEmail> authors = new HashSet<>(prevAuthors);
+
+ Set<NameAndEmail> prevCommitters;
+ bytes = getBytes(defaultStore.get(txn, COMMITTERS_KEY));
+ if (bytes != null)
+ prevCommitters = (Set<NameAndEmail>) SerializationUtils.deserialize(bytes);
+ else
+ prevCommitters = new HashSet<>();
+ Set<NameAndEmail> committers = new HashSet<>(prevCommitters);
+
+ Map<String, Long> files;
+ bytes = getBytes(defaultStore.get(txn, FILES_KEY));
+ if (bytes != null)
+ files = (Map<String, Long>) SerializationUtils.deserialize(bytes);
+ else
+ files = new HashMap<>();
+
+ /*
+ * Use a synchronous queue to achieve below purpose:
+ * 1. Add commit to Xodus transactional store in the same thread opening the transaction
+ * as this is required by Xodus
+ * 2. Do not pile up commits to use minimal memory
+ */
+ SynchronousQueue<Optional<LogCommit>> queue = new SynchronousQueue<>();
+ AtomicReference<Exception> logException = new AtomicReference<>(null);
+ executorService.execute(new Runnable() {
+
+ @Override
+ public void run() {
+ try {
+ List<String> revisions = new ArrayList<>();
+ revisions.add(commitId.name());
+ if (lastCommitId != null)
+ revisions.add("^" + lastCommitId.name());
+
+ new LogCommand(project.getGitDir()) {
+
+ @Override
+ protected void consume(LogCommit commit) {
+ try {
+ queue.put(Optional.of(commit));
+ } catch (InterruptedException e) {
+ }
+ }
+
+ }.revisions(revisions).call();
+
+ } catch (Exception e) {
+ logException.set(e);
+ } finally {
+ try {
+ queue.put(Optional.empty());
+ } catch (InterruptedException e) {
+ }
+ }
}
- commitsStore.put(txn, key, new ArrayByteIterable(newValueBytes));
- newCommitCount++;
+ });
+
+ Optional<LogCommit> commitOptional = queue.take();
+ while (commitOptional.isPresent()) {
+ LogCommit commit = commitOptional.get();
- for (RevCommit parent: commit.getParents()) {
- keyBytes = new byte[20];
- parent.copyRawTo(keyBytes, 0);
- key = new ArrayByteIterable(keyBytes);
- valueBytes = getBytes(commitsStore.get(txn, key));
- if (valueBytes != null) {
- newValueBytes = new byte[valueBytes.length+20];
- System.arraycopy(valueBytes, 0, newValueBytes, 0, valueBytes.length);
+ byte[] keyBytes = new byte[20];
+ ObjectId.fromString(commit.getHash()).copyRawTo(keyBytes, 0);
+ ByteIterable key = new ArrayByteIterable(keyBytes);
+ byte[] valueBytes = getBytes(commitsStore.get(txn, key));
+
+ if (valueBytes == null || valueBytes.length % 20 == 0) {
+ /*
+ * Length of stored bytes of a commit is either 20*nChild
+ * (20 is length of ObjectId), or 1+20*nChild, as we need
+ * an extra leading byte to differentiate commits being
+ * processed and commits with child information attached
+ * but not processed.
+ */
+ byte[] newValueBytes;
+ if (valueBytes == null) {
+ newValueBytes = new byte[1];
} else {
- newValueBytes = new byte[20];
+ newValueBytes = new byte[valueBytes.length+1];
+ System.arraycopy(valueBytes, 0, newValueBytes, 1, valueBytes.length);
}
- commit.copyRawTo(newValueBytes, newValueBytes.length-20);
commitsStore.put(txn, key, new ArrayByteIterable(newValueBytes));
- }
-
- if (StringUtils.isNotBlank(commit.getAuthorIdent().getName())
- || StringUtils.isNotBlank(commit.getAuthorIdent().getEmailAddress())) {
- authors.add(new NameAndEmail(commit.getAuthorIdent()));
- }
+
+ newCommitCount++;
+
+ for (String parentHash: commit.getParentHashes()) {
+ keyBytes = new byte[20];
+ ObjectId.fromString(parentHash).copyRawTo(keyBytes, 0);
+ key = new ArrayByteIterable(keyBytes);
+ valueBytes = getBytes(commitsStore.get(txn, key));
+ if (valueBytes != null) {
+ newValueBytes = new byte[valueBytes.length+20];
+ System.arraycopy(valueBytes, 0, newValueBytes, 0, valueBytes.length);
+ } else {
+ newValueBytes = new byte[20];
+ }
+ ObjectId.fromString(commit.getHash()).copyRawTo(newValueBytes, newValueBytes.length-20);
+ commitsStore.put(txn, key, new ArrayByteIterable(newValueBytes));
+ }
+
+ if (commit.getCommitter() != null) {
+ committers.add(new NameAndEmail(commit.getCommitter()));
+ for (String file: commit.getChangedFiles())
+ files.put(file, commit.getCommitter().getWhen().getTime());
+ }
- if (StringUtils.isNotBlank(commit.getCommitterIdent().getName())
- || StringUtils.isNotBlank(commit.getCommitterIdent().getEmailAddress())) {
- committers.add(new NameAndEmail(commit.getCommitterIdent()));
- }
-
- String authorEmail = commit.getAuthorIdent().getEmailAddress();
- if (StringUtils.isNotBlank(authorEmail) && commit.getParentCount() == 1) {
- try (TreeWalk treeWalk = new TreeWalk(project.getRepository())) {
- treeWalk.setFilter(TreeFilter.ANY_DIFF);
- treeWalk.addTree(commit.getTree());
- treeWalk.addTree(commit.getParent(0).getTree());
- treeWalk.setRecursive(true);
- while (treeWalk.next()) {
- String path = treeWalk.getPathString();
- files.add(path);
-
- updateContribution(txn, contributionsStore, authorEmail, path);
+ if (commit.getAuthor() != null) {
+ authors.add(new NameAndEmail(commit.getAuthor()));
+ for (String path: commit.getChangedFiles()) {
+ updateContribution(txn, contributionsStore, commit.getAuthor().getEmailAddress(), path);
while (path.contains("/")) {
path = StringUtils.substringBeforeLast(path, "/");
- updateContribution(txn, contributionsStore, authorEmail, path);
+ updateContribution(txn, contributionsStore,
+ commit.getAuthor().getEmailAddress(), path);
}
- updateContribution(txn, contributionsStore, authorEmail, "");
+ updateContribution(txn, contributionsStore,
+ commit.getAuthor().getEmailAddress(), "");
}
}
- }
- }
- commit = revWalk.next();
+ }
+ commitOptional = queue.take();
+ }
+ if (logException.get() != null)
+ throw logException.get();
+
+ if (newCommitCount != prevCommitCount) {
+ bytes = SerializationUtils.serialize(newCommitCount);
+ defaultStore.put(txn, COMMIT_COUNT_KEY, new ArrayByteIterable(bytes));
+ commitCountCache.put(project.getId(), newCommitCount);
+ }
+
+ if (!authors.equals(prevAuthors)) {
+ bytes = SerializationUtils.serialize((Serializable) authors);
+ defaultStore.put(txn, AUTHORS_KEY, new ArrayByteIterable(bytes));
+ authorsCache.remove(project.getId());
+ }
+
+ if (!committers.equals(prevCommitters)) {
+ bytes = SerializationUtils.serialize((Serializable) committers);
+ defaultStore.put(txn, COMMITTERS_KEY, new ArrayByteIterable(bytes));
+ committersCache.remove(project.getId());
+ }
+
+ if (files.size() > MAX_COLLECTING_FILES) {
+ List<String> fileList = new ArrayList<>(files.keySet());
+ fileList.sort((file1, file2)->files.get(file1).compareTo(files.get(file2)));
+ for (int i=0; i<fileList.size() - MAX_COLLECTING_FILES; i++)
+ files.remove(fileList.get(i));
+ }
+ bytes = SerializationUtils.serialize((Serializable) files);
+ defaultStore.put(txn, FILES_KEY, new ArrayByteIterable(bytes));
+ filesCache.remove(project.getId());
+
+ bytes = new byte[20];
+ commitId.copyRawTo(bytes, 0);
+ defaultStore.put(txn, LAST_COMMIT_KEY, new ArrayByteIterable(bytes));
+ } catch (Exception e) {
+ Throwables.propagate(e);
}
-
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
-
- if (newCommitCount != prevCommitCount) {
- bytes = SerializationUtils.serialize(newCommitCount);
- defaultStore.put(txn, COMMIT_COUNT_KEY, new ArrayByteIterable(bytes));
- commitCountCache.put(project.getId(), newCommitCount);
- }
-
- if (!authors.equals(prevAuthors)) {
- bytes = SerializationUtils.serialize((Serializable) authors);
- defaultStore.put(txn, AUTHORS_KEY, new ArrayByteIterable(bytes));
- authorsCache.remove(project.getId());
- }
-
- if (!committers.equals(prevCommitters)) {
- bytes = SerializationUtils.serialize((Serializable) committers);
- defaultStore.put(txn, COMMITTERS_KEY, new ArrayByteIterable(bytes));
- committersCache.remove(project.getId());
- }
-
- if (!files.equals(prevFiles)) {
- bytes = SerializationUtils.serialize((Serializable) files);
- defaultStore.put(txn, FILES_KEY, new ArrayByteIterable(bytes));
- filesCache.remove(project.getId());
}
- bytes = new byte[20];
- commits.get(commits.size()-1).copyRawTo(bytes, 0);
- defaultStore.put(txn, LAST_COMMIT_KEY, new ArrayByteIterable(bytes));
- }
-
- });
- logger.debug("Commits info collected (project: {}, commits: {})", project, commits.size());
+ });
+ }
}
private void updateContribution(Transaction txn, Store contributionsStore, String email, String path) {
@@ -385,7 +457,8 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme
public List<String> compute(Transaction txn) {
byte[] bytes = getBytes(store.get(txn, FILES_KEY));
if (bytes != null) {
- List<String> files = new ArrayList<>((Set<String>)SerializationUtils.deserialize(bytes));
+ List<String> files = new ArrayList<>(
+ ((Map<String, Long>)SerializationUtils.deserialize(bytes)).keySet());
Map<String, List<String>> segmentsMap = new HashMap<>();
Splitter splitter = Splitter.on("/");
for (String file: files) {
@@ -457,7 +530,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme
current.copyRawTo(keyBytes, 0);
byte[] valueBytes = getBytes(store.get(txn, new ArrayByteIterable(keyBytes)));
if (valueBytes != null) {
- if (valueBytes.length % 2 == 0) {
+ if (valueBytes.length % 20 == 0) {
for (int i=0; i<valueBytes.length/20; i++) {
ObjectId child = ObjectId.fromRaw(valueBytes, i*20);
if (!descendants.contains(child)) {
@@ -502,7 +575,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme
parent.copyRawTo(keyBytes, 0);
byte[] valueBytes = getBytes(store.get(txn, new ArrayByteIterable(keyBytes)));
if (valueBytes != null) {
- if (valueBytes.length % 2 == 0) {
+ if (valueBytes.length % 20 == 0) {
for (int i=0; i<valueBytes.length/20; i++) {
children.add(ObjectId.fromRaw(valueBytes, i*20));
}
@@ -551,14 +624,22 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme
@Override
public Void call() throws Exception {
Project project = projectManager.load(projectId);
- List<RevCommit> commits = new ArrayList<>();
- try (RevWalk revWalk = new RevWalk(project.getRepository())) {
- for (Prioritized work: works) {
- CollectingWork collectingWork = (CollectingWork) work;
- commits.add(revWalk.parseCommit(collectingWork.getCommitId()));
+ List<ObjectId> commitIds = new ArrayList<>();
+ if (works.size() > 1) {
+ try (RevWalk revWalk = new RevWalk(project.getRepository())) {
+ List<RevCommit> commits = new ArrayList<>();
+ for (Object work: works)
+ commits.add(revWalk.parseCommit(((CollectingWork)work).getCommitId()));
+ commits.sort(Comparator.comparing(RevCommit::getCommitTime));
+ for (RevCommit commit: commits)
+ commitIds.add(commit.copy());
}
+ } else {
+ CollectingWork collectingWork = (CollectingWork) works.iterator().next();
+ commitIds.add(collectingWork.getCommitId());
}
- doCollect(project, commits);
+ for (ObjectId commitId: commitIds)
+ doCollect(project, commitId, true);
return null;
}
@@ -589,7 +670,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme
commit.copyRawTo(keyBytes, 0);
ByteIterable key = new ArrayByteIterable(keyBytes);
byte[] valueBytes = getBytes(commitsStore.get(txn, key));
- return valueBytes != null && valueBytes.length%2 != 0;
+ return valueBytes != null && valueBytes.length % 20 != 0;
}
});
@@ -603,7 +684,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme
commits.sort(Comparator.comparing(RevCommit::getCommitTime));
for (RevCommit commit: commits) {
- CollectingWork work = new CollectingWork(PRIORITY, commit.getId());
+ CollectingWork work = new CollectingWork(PRIORITY, commit.copy());
batchWorkManager.submit(getBatchWorker(project), work);
}
}
@@ -620,11 +701,8 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme
@Listen
public void on(RefUpdated event) {
if (!event.getNewObjectId().equals(ObjectId.zeroId())) {
- RevCommit commit = event.getProject().getRevCommit(event.getNewObjectId(), false);
- if (commit != null) {
- CollectingWork work = new CollectingWork(PRIORITY, commit.getId());
- batchWorkManager.submit(getBatchWorker(event.getProject()), work);
- }
+ CollectingWork work = new CollectingWork(PRIORITY, event.getNewObjectId());
+ batchWorkManager.submit(getBatchWorker(event.getProject()), work);
}
}
diff --git a/web/src/main/java/com/gitplex/server/web/component/diff/blob/text/TextDiffPanel.java b/web/src/main/java/com/gitplex/server/web/component/diff/blob/text/TextDiffPanel.java
index 191fbea532..115dc0671f 100644
--- a/web/src/main/java/com/gitplex/server/web/component/diff/blob/text/TextDiffPanel.java
+++ b/web/src/main/java/com/gitplex/server/web/component/diff/blob/text/TextDiffPanel.java
@@ -45,7 +45,7 @@ import com.gitplex.server.GitPlex;
import com.gitplex.server.git.Blame;
import com.gitplex.server.git.BlobChange;
import com.gitplex.server.git.BlobIdent;
-import com.gitplex.server.git.Commit;
+import com.gitplex.server.git.BlameCommit;
import com.gitplex.server.git.GitUtils;
import com.gitplex.server.git.command.BlameCommand;
import com.gitplex.server.manager.CodeCommentManager;
@@ -705,7 +705,7 @@ public class TextDiffPanel extends Panel implements SourceAware {
}
private void appendBlame(StringBuilder builder, int oldLineNo, int newLineNo) {
- Commit commit;
+ BlameCommit commit;
if (newLineNo != -1)
commit = Preconditions.checkNotNull(blameInfo.newBlame.get(newLineNo));
else
@@ -1208,9 +1208,9 @@ public class TextDiffPanel extends Panel implements SourceAware {
private static class BlameInfo implements Serializable {
- Map<Integer, Commit> oldBlame = new HashMap<>();
+ Map<Integer, BlameCommit> oldBlame = new HashMap<>();
- Map<Integer, Commit> newBlame = new HashMap<>();
+ Map<Integer, BlameCommit> newBlame = new HashMap<>();
String lastCommitHash;
diff --git a/web/src/main/java/com/gitplex/server/web/page/base/base.js b/web/src/main/java/com/gitplex/server/web/page/base/base.js
index 436ad14376..fcc0184111 100644
--- a/web/src/main/java/com/gitplex/server/web/page/base/base.js
+++ b/web/src/main/java/com/gitplex/server/web/page/base/base.js
@@ -248,23 +248,30 @@ gitplex.server = {
e.stopPropagation();
});
+ var ajaxCalls = 0;
Wicket.Event.subscribe('/ajax/call/beforeSend', function() {
- var $ajaxLoadingIndicator = $("#ajax-loading-indicator");
- if ($ajaxLoadingIndicator[0].timer)
- clearTimeout($ajaxLoadingIndicator[0].timer);
- $ajaxLoadingIndicator[0].timer = setTimeout(function() {
- if (!$ajaxLoadingIndicator.is(":visible") && $(".ajax-indicator:visible").length == 0)
- $ajaxLoadingIndicator.show();
- }, 2000);
+ if (ajaxCalls == 0) {
+ var $ajaxLoadingIndicator = $("#ajax-loading-indicator");
+ if ($ajaxLoadingIndicator[0].timer)
+ clearTimeout($ajaxLoadingIndicator[0].timer);
+ $ajaxLoadingIndicator[0].timer = setTimeout(function() {
+ if (!$ajaxLoadingIndicator.is(":visible") && $(".ajax-indicator:visible").length == 0)
+ $ajaxLoadingIndicator.show();
+ }, 2000);
+ }
+ ajaxCalls++;
});
- Wicket.Event.subscribe('/ajax/call/complete', function() {
- var $ajaxLoadingIndicator = $("#ajax-loading-indicator");
- if ($ajaxLoadingIndicator[0].timer) {
- clearTimeout($ajaxLoadingIndicator[0].timer);
- $ajaxLoadingIndicator[0].timer = null;
+ Wicket.Event.subscribe('/ajax/call/done', function() {
+ ajaxCalls--;
+ if (ajaxCalls == 0) {
+ var $ajaxLoadingIndicator = $("#ajax-loading-indicator");
+ if ($ajaxLoadingIndicator[0].timer) {
+ clearTimeout($ajaxLoadingIndicator[0].timer);
+ $ajaxLoadingIndicator[0].timer = null;
+ }
+ $ajaxLoadingIndicator.hide();
}
- $ajaxLoadingIndicator.hide();
});
},
diff --git a/web/src/main/java/com/gitplex/server/web/page/project/commit/ProjectCommitsPage.java b/web/src/main/java/com/gitplex/server/web/page/project/commit/ProjectCommitsPage.java
index d5d4b1c279..a6736c251c 100644
--- a/web/src/main/java/com/gitplex/server/web/page/project/commit/ProjectCommitsPage.java
+++ b/web/src/main/java/com/gitplex/server/web/page/project/commit/ProjectCommitsPage.java
@@ -112,7 +112,6 @@ public class ProjectCommitsPage extends ProjectPage {
@Override
protected Commits load() {
Commits commits = new Commits();
-
List<String> commitHashes;
try {
RevListCommand command = new RevListCommand(getProject().getGitDir());