The Unified and Autonomous Development Platform
Merge branch 'master' into stats
robin shenabe16dbda659
diff --cc core/src/main/java/com/gitplex/server/git/command/FileChange.javaindex b09a4fd649,bcfa169f6e..c3e1cde564--- a/core/src/main/java/com/gitplex/server/git/command/FileChange.java+++ b/core/src/main/java/com/gitplex/server/git/command/FileChange.java@@@ -13,9 -17,11 +13,9 @@@ public class FileChange implements Seriprivate final int deletions;- public FileChange(String oldPath, String newPath, int additions, int deletions) {- public FileChange(Action action, String oldPath, String newPath, int additions, int deletions) {- this.action = action;-++ public FileChange(String oldPath, String path, int additions, int deletions) {this.oldPath = oldPath;- this.path = newPath;- this.newPath = newPath;++ this.path = path;this.additions = additions;this.deletions = deletions;}diff --cc core/src/main/java/com/gitplex/server/git/command/GitCommit.javaindex a4446e5b3b,7e10d1e03d..4e0234b31e--- a/core/src/main/java/com/gitplex/server/git/command/GitCommit.java+++ b/core/src/main/java/com/gitplex/server/git/command/GitCommit.java@@@ -74,25 -70,6 +74,29 @@@ public class GitCommit implements Seriareturn author;}+ @Nullable+ public Date getCommitDate() {+ return commitDate;+ }++ public int getAdditions() {+ int additions = 0;- for (FileChange change: fileChanges)- additions += change.getAdditions();++ for (FileChange change: fileChanges) {++ if (change.getAdditions() > 0)++ additions += change.getAdditions();++ }+ return additions;+ }++ public int getDeletions() {+ int deletions = 0;- for (FileChange change: fileChanges)++ for (FileChange change: fileChanges) {++ if (change.getDeletions() > 0)+ deletions += change.getDeletions();++ }+ return deletions;+ }+public static class Builder {public String hash;diff --cc core/src/main/java/com/gitplex/server/git/command/ListFileChangesCommand.javaindex 0000000000,ea6c97ab5d..482e57c888mode 000000,100644..100644--- a/core/src/main/java/com/gitplex/server/git/command/ListFileChangesCommand.java+++ b/core/src/main/java/com/gitplex/server/git/command/ListFileChangesCommand.java@@@ -1,0 -1,99 +1,98 @@@+ package com.gitplex.server.git.command;++ import java.io.File;+ import java.util.ArrayList;+ import java.util.Collection;+ import java.util.StringTokenizer;++ import org.eclipse.jgit.util.QuotedString;+ import org.slf4j.Logger;+ import org.slf4j.LoggerFactory;+-import com.gitplex.server.git.command.FileChange.Action;+ import com.gitplex.utils.command.Commandline;+ import com.gitplex.utils.command.LineConsumer;+ import com.google.common.base.Preconditions;++ public class ListFileChangesCommand extends GitCommand<Collection<FileChange>> {++ private static final Logger logger = LoggerFactory.getLogger(ListFileChangesCommand.class);++ private String fromRev;++ private String toRev;++ private String path;++ public ListFileChangesCommand(File gitDir) {+ super(gitDir);+ }++ public ListFileChangesCommand fromRev(String fromRev) {+ this.fromRev = fromRev;+ return this;+ }++ public ListFileChangesCommand toRev(String toRev) {+ this.toRev = toRev;+ return this;+ }++ public ListFileChangesCommand path(String path) {+ this.path = path;+ return this;+ }++ @Override+ public Collection<FileChange> call() {+ Preconditions.checkNotNull(toRev, "toRev has to be specified.");+ Preconditions.checkNotNull(fromRev, "fromRev has to be specified.");++ final Collection<FileChange> fileChanges = new ArrayList<>();++ Commandline cmd = cmd();++ cmd.addArgs("diff", "--numstat", "--no-renames", fromRev + ".." + toRev);++ if (path != null)+ cmd.addArgs("--", path);++ cmd.execute(new LineConsumer() {++ @Override+ public void consume(String line) {+ if (line.contains("\t")) {+ StringTokenizer tokenizer = new StringTokenizer(line, "\t");++ int additions;+ String field = tokenizer.nextToken();+ if (field.equals("-"))+ additions = -1;+ else+ additions = Integer.parseInt(field);++ int deletions;+ field = tokenizer.nextToken();+ if (field.equals("-"))+ deletions = -1;+ else+ deletions = Integer.parseInt(field);++ String file = QuotedString.GIT_PATH.dequote(tokenizer.nextToken());+- fileChanges.add(new FileChange(Action.MODIFY, file, file, additions, deletions));++ fileChanges.add(new FileChange(null, file, additions, deletions));+ }+ }++ }, new LineConsumer() {++ @Override+ public void consume(String line) {+ logger.error(line);+ }++ }).checkReturnCode();++ return fileChanges;+ }++ }diff --cc core/src/main/java/com/gitplex/server/git/command/LogCommand.javaindex 0ed2cc538a,ecc681b4d6..94cc8892a5--- a/core/src/main/java/com/gitplex/server/git/command/LogCommand.java+++ b/core/src/main/java/com/gitplex/server/git/command/LogCommand.java@@@ -115,38 -110,7 +115,38 @@@ public abstract class LogCommand extend} else if (line.startsWith("authorDate:")) {commitBuilderRef.get().authorDate =GitUtils.parseRawDate(line.substring("authorDate:".length()).trim());- inFileChangesBlock.set(true);+ } else if (line.trim().length() != 0) {+ StringTokenizer tokenizer = new StringTokenizer(line, "\t");+ String additionsToken = tokenizer.nextToken();- int additions = additionsToken.equals("-")?0:Integer.parseInt(additionsToken);++ int additions = additionsToken.equals("-")?-1:Integer.parseInt(additionsToken);+ String deletionsToken = tokenizer.nextToken();- int deletions = deletionsToken.equals("-")?0:Integer.parseInt(deletionsToken);++ int deletions = deletionsToken.equals("-")?-1:Integer.parseInt(deletionsToken);++ String path = tokenizer.nextToken();+ int renameSignIndex = path.indexOf(" => ");+ if (renameSignIndex != -1) {+ int leftBraceIndex = path.indexOf("{");+ int rightBraceIndex = path.indexOf("}");+ if (leftBraceIndex != -1 && rightBraceIndex != -1 && leftBraceIndex<renameSignIndex+ && rightBraceIndex>renameSignIndex) {+ String leftCommon = path.substring(0, leftBraceIndex);+ String rightCommon = path.substring(rightBraceIndex+1);+ String oldPath = leftCommon + path.substring(leftBraceIndex+1, renameSignIndex)+ + rightCommon;+ String newPath = leftCommon + path.substring(renameSignIndex+4, rightBraceIndex)+ + rightCommon;+ commitBuilderRef.get().fileChanges.add(+ new FileChange(oldPath, newPath, additions, deletions));+ } else {+ String oldPath = QuotedString.GIT_PATH.dequote(path.substring(0, renameSignIndex));+ String newPath = QuotedString.GIT_PATH.dequote(path.substring(renameSignIndex+4));+ commitBuilderRef.get().fileChanges.add(+ new FileChange(oldPath, newPath, additions, deletions));+ }+ } else {+ path = QuotedString.GIT_PATH.dequote(path);+ commitBuilderRef.get().fileChanges.add(new FileChange(null, path, additions, deletions));+ }}}diff --cc core/src/main/java/com/gitplex/server/manager/impl/DefaultCommitInfoManager.javaindex 19d39fc9ad,5fb57fee32..360e68f307--- a/core/src/main/java/com/gitplex/server/manager/impl/DefaultCommitInfoManager.java+++ b/core/src/main/java/com/gitplex/server/manager/impl/DefaultCommitInfoManager.java@@@ -522,210 -270,137 +522,212 @@@ public class DefaultCommitInfoManager e} catch (InterruptedException e) {}}- }-- });-- Optional<LogCommit> commitOptional = queue.take();- while (commitOptional.isPresent()) {- LogCommit commit = commitOptional.get();-- byte[] keyBytes = new byte[20];- ObjectId.fromString(commit.getHash()).copyRawTo(keyBytes, 0);- ByteIterable key = new ArrayByteIterable(keyBytes);- byte[] valueBytes = getBytes(commitsStore.get(txn, key));-- if (valueBytes == null || valueBytes.length % 20 == 0) {- /*- * Length of stored bytes of a commit is either 20*nChild- * (20 is length of ObjectId), or 1+20*nChild, as we need- * an extra leading byte to differentiate commits being- * processed and commits with child information attached- * but not processed.- */- byte[] newValueBytes;- if (valueBytes == null) {- newValueBytes = new byte[1];- } else {- newValueBytes = new byte[valueBytes.length+1];- System.arraycopy(valueBytes, 0, newValueBytes, 1, valueBytes.length);- }- commitsStore.put(txn, key, new ArrayByteIterable(newValueBytes));-- newCommitCount++;- for (String parentHash: commit.getParentHashes()) {- keyBytes = new byte[20];- ObjectId.fromString(parentHash).copyRawTo(keyBytes, 0);- key = new ArrayByteIterable(keyBytes);- valueBytes = getBytes(commitsStore.get(txn, key));- if (valueBytes != null) {- newValueBytes = new byte[valueBytes.length+20];- System.arraycopy(valueBytes, 0, newValueBytes, 0, valueBytes.length);- } else {- newValueBytes = new byte[20];+ }.revisions(revisions);++ executorService.execute(new Runnable() {++ @Override+ public void run() {+ try {+ log.call();+ } catch (Exception e) {+ logException.set(e);+ } finally {+ try {+ queue.put(Optional.empty());+ } catch (InterruptedException e) {+ }}- ObjectId.fromString(commit.getHash()).copyRawTo(newValueBytes, newValueBytes.length-20);- commitsStore.put(txn, key, new ArrayByteIterable(newValueBytes));}- if (commit.getCommitter() != null) {- committers.add(new NameAndEmail(commit.getCommitter()));- for (String file: commit.getChangedFiles())- files.put(file, commit.getCommitter().getWhen().getTime());- }+ });++ try {+ Optional<GitCommit> logCommitOptional = queue.take();+ while (logCommitOptional.isPresent()) {+ GitCommit logCommit = logCommitOptional.get();++ if (logCommit.getCommitDate() != null) {+ for (String file: logCommit.getChangedFiles())+ files.put(file, logCommit.getCommitDate().getTime());+ }++ if (logCommit.getCommitter() != null)+ users.add(new NameAndEmail(logCommit.getCommitter()));- if (commit.getAuthor() != null) {- authors.add(new NameAndEmail(commit.getAuthor()));- for (String changedFile: commit.getChangedFiles()) {- updateContribution(txn, contributionsStore,- commit.getAuthor().getEmailAddress(), changedFile);- while (changedFile.contains("/")) {- changedFile = StringUtils.substringBeforeLast(changedFile, "/");- updateContribution(txn, contributionsStore,- commit.getAuthor().getEmailAddress(), changedFile);+ if (logCommit.getAuthor() != null) {+ NameAndEmail nameAndEmail = new NameAndEmail(logCommit.getAuthor());+ users.add(nameAndEmail);++ String emailAddress = logCommit.getAuthor().getEmailAddress();+ if (StringUtils.isNotBlank(emailAddress)) {+ ByteIterable emailKey = new StringByteIterable(emailAddress);+ int userIndex = readInt(emailToIndexStore, txn, emailKey, -1);+ if (userIndex == -1) {+ userIndex = nextIndex.user++;+ writeInt(emailToIndexStore, txn, emailKey, userIndex);+ indexToUserStore.put(txn,+ new IntByteIterable(userIndex),+ new ArrayByteIterable(SerializationUtils.serialize(nameAndEmail)));+ }++ if (logCommit.getCommitDate() != null && logCommit.getParentHashes().size() <= 1) {+ int dayValue = new Day(logCommit.getCommitDate()).getValue();+ Map<Integer, Contribution> contributionsOnDay =+ dailyContributionsCache.get(dayValue);+ if (contributionsOnDay == null) {+ contributionsOnDay = deserializeContributions(readBytes(+ dailyContributionsStore, txn, new IntByteIterable(dayValue)));+ dailyContributionsCache.put(dayValue, contributionsOnDay);+ }+ updateContribution(contributionsOnDay, userIndex, logCommit);+ }++ for (FileChange change: logCommit.getFileChanges()) {+ String path = change.getPath();+ int pathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,+ nextIndex, path);+ int edits = change.getAdditions() + change.getDeletions();++ if (edits < 0)++ edits = 100;+ updateEdits(editsStore, txn, editsCache, userIndex, pathIndex, edits);+ while (path.contains("/")) {+ path = StringUtils.substringBeforeLast(path, "/");+ pathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,+ nextIndex, path);+ updateEdits(editsStore, txn, editsCache, userIndex, pathIndex, edits);+ }+ pathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,+ nextIndex, "");+ updateEdits(editsStore, txn, editsCache, userIndex, pathIndex, edits);+ }}- updateContribution(txn, contributionsStore,- commit.getAuthor().getEmailAddress(), "");}- for (FileChange change: commit.getFileChanges()) {- if (change.getAction() == FileChange.Action.COPY- || change.getAction() == FileChange.Action.RENAME) {- ByteIterable renameKey = new StringByteIterable(change.getNewPath());- List<String> renamedFroms;- valueBytes = getBytes(renamesStore.get(txn, renameKey));- if (valueBytes != null) {- renamedFroms = (List<String>) SerializationUtils.deserialize(valueBytes);++ if (logCommit.getCommitDate() != null && logCommit.getParentHashes().size() <= 1) {+ int dayValue = new Day(logCommit.getCommitDate()).getValue();+ updateContribution(overallContributions, dayValue, logCommit);+ }++ for (FileChange change: logCommit.getFileChanges()) {+ if (change.getOldPath() != null) {+ int pathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,+ nextIndex, change.getPath());+ ByteIterable pathKey = new IntByteIterable(pathIndex);+ Set<Integer> historyPathIndexes = new HashSet<>();+ byte[] bytesOfHistoryPaths = readBytes(historyPathsStore, txn, pathKey);+ if (bytesOfHistoryPaths == null) {+ bytesOfHistoryPaths = new byte[0];+ int pos = 0;+ for (int i=0; i<bytesOfHistoryPaths.length/Integer.SIZE; i++) {+ historyPathIndexes.add(ByteBuffer.wrap(bytesOfHistoryPaths, pos, Integer.SIZE).getInt());+ pos += Integer.SIZE;+ }} else {- renamedFroms = new ArrayList<>();+ historyPathIndexes = new HashSet<>();}- if (!renamedFroms.contains(change.getOldPath())) {- renamedFroms.add(0, change.getOldPath());- if (renamedFroms.size() > MAX_HISTORY_PATHS)- renamedFroms.remove(renamedFroms.size()-1);+ if (historyPathIndexes.size() < MAX_HISTORY_PATHS) {+ int oldPathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,+ nextIndex, change.getOldPath());+ if (!historyPathIndexes.contains(oldPathIndex)) {+ historyPathIndexes.add(oldPathIndex);+ byte[] newBytesOfHistoryPaths =+ new byte[bytesOfHistoryPaths.length+Integer.SIZE];+ System.arraycopy(bytesOfHistoryPaths, 0,+ newBytesOfHistoryPaths, 0, bytesOfHistoryPaths.length);+ ByteBuffer buffer = ByteBuffer.wrap(newBytesOfHistoryPaths,+ bytesOfHistoryPaths.length, Integer.BYTES);+ buffer.putInt(oldPathIndex);+ historyPathsStore.put(txn, pathKey,+ new ArrayByteIterable(newBytesOfHistoryPaths));+ }}-- valueBytes = SerializationUtils.serialize((Serializable) renamedFroms);- renamesStore.put(txn, renameKey, new ArrayByteIterable(valueBytes));}+ }++ if (logCommit.getCommitDate() != null && logCommit.getParentHashes().size() <= 1) {+ int day = new Day(logCommit.getCommitDate()).getValue();++ Map<String, Integer> lineStatsOnDay = lineStats.get(day);+ if (lineStatsOnDay == null) {+ lineStatsOnDay = new HashMap<>();+ lineStats.put(day, lineStatsOnDay);+ }++ Map<String, Integer> languageLines = new HashMap<>();+ for (FileChange change: logCommit.getFileChanges()) {+ int lines = change.getAdditions() - change.getDeletions();+ int lastIndexOfDot = change.getPath().lastIndexOf('.');+ if (lastIndexOfDot != -1 && lines != 0) {+ String fileExt = change.getPath().substring(lastIndexOfDot+1).toLowerCase();+ String language = PROGRAMMING_LANGUAGES.get(fileExt);+ if (language != null) {+ Integer accumulatedLines = languageLines.get(language);+ if (accumulatedLines != null)+ lines += accumulatedLines;+ languageLines.put(language, lines);+ }+ }+ }++ for (Map.Entry<String, Integer> entry: languageLines.entrySet()) {+ String language = entry.getKey();+ Integer lines = entry.getValue();+ Integer accumulatedLines = lineStatsOnDay.get(language);+ if (accumulatedLines != null)+ lines += accumulatedLines;+ lineStatsOnDay.put(language, lines);+ }+}++ logCommitOptional = queue.take();}- }- commitOptional = queue.take();- }- if (logException.get() != null)- throw logException.get();-- if (newCommitCount != prevCommitCount) {- bytes = SerializationUtils.serialize(newCommitCount);- defaultStore.put(txn, COMMIT_COUNT_KEY, new ArrayByteIterable(bytes));- commitCountCache.put(project.getId(), newCommitCount);- }-- if (!authors.equals(prevAuthors)) {- bytes = SerializationUtils.serialize((Serializable) authors);- defaultStore.put(txn, AUTHORS_KEY, new ArrayByteIterable(bytes));- authorsCache.remove(project.getId());- }-- if (!committers.equals(prevCommitters)) {- bytes = SerializationUtils.serialize((Serializable) committers);- defaultStore.put(txn, COMMITTERS_KEY, new ArrayByteIterable(bytes));- committersCache.remove(project.getId());- }-- if (files.size() > MAX_COLLECTING_FILES) {- List<String> fileList = new ArrayList<>(files.keySet());- fileList.sort((file1, file2)->files.get(file1).compareTo(files.get(file2)));- for (int i=0; i<fileList.size() - MAX_COLLECTING_FILES; i++)- files.remove(fileList.get(i));+ if (logException.get() != null)+ throw logException.get();+ } catch (Exception e) {+ Throwables.propagate(e);+ }++ for (Map.Entry<Integer, Map<Integer, Contribution>> entry: dailyContributionsCache.entrySet()) {+ byte[] bytesOfContributionsOnDay = serializeContributions(entry.getValue());+ dailyContributionsStore.put(txn, new IntByteIterable(entry.getKey()),+ new ArrayByteIterable(bytesOfContributionsOnDay));+ }+ defaultStore.put(txn, OVERALL_CONTRIBUTIONS_KEY,+ new ArrayByteIterable(serializeContributions(overallContributions)));++ bytesOfLineStats = SerializationUtils.serialize((Serializable) lineStats);+ defaultStore.put(txn, LINE_STATS_KEY, new ArrayByteIterable(bytesOfLineStats));++ writeInt(defaultStore, txn, NEXT_USER_INDEX_KEY, nextIndex.user);+ writeInt(defaultStore, txn, NEXT_PATH_INDEX_KEY, nextIndex.path);++ userBytes = SerializationUtils.serialize((Serializable) users);+ defaultStore.put(txn, USERS_KEY, new ArrayByteIterable(userBytes));+ usersCache.remove(project.getId());++ if (files.size() > MAX_COLLECTING_FILES) {+ List<String> fileList = new ArrayList<>(files.keySet());+ fileList.sort((file1, file2)->files.get(file1).compareTo(files.get(file2)));+ for (int i=0; i<fileList.size() - MAX_COLLECTING_FILES; i++)+ files.remove(fileList.get(i));+ }+ fileBytes = SerializationUtils.serialize((Serializable) files);+ defaultStore.put(txn, FILES_KEY, new ArrayByteIterable(fileBytes));+ filesCache.remove(project.getId());++ for (Map.Entry<Long, Integer> entry: editsCache.entrySet())+ writeInt(editsStore, txn, new LongByteIterable(entry.getKey()), entry.getValue());++ byte[] bytes = new byte[20];+ currentCommitId.copyRawTo(bytes, 0);+ defaultStore.put(txn, LAST_COMMIT_OF_DEFAULT_BRANCH_KEY, new ArrayByteIterable(bytes));}- bytes = SerializationUtils.serialize((Serializable) files);- defaultStore.put(txn, FILES_KEY, new ArrayByteIterable(bytes));- filesCache.remove(project.getId());-- bytes = new byte[20];- commitId.copyRawTo(bytes, 0);- defaultStore.put(txn, LAST_COMMIT_KEY, new ArrayByteIterable(bytes));- } catch (Exception e) {- Throwables.propagate(e);- }+ });}-- });+ }}}diff --cc core/src/main/java/com/gitplex/server/util/QualityCheckStatus.javaindex 18567236c5,d9985082b0..6981ddf3dc--- a/core/src/main/java/com/gitplex/server/util/QualityCheckStatus.java+++ b/core/src/main/java/com/gitplex/server/util/QualityCheckStatus.java@@@ -274,20 -276,29 +276,29 @@@ public class QualityCheckStatus}private void sortByContributions(List<UserFacade> users, PullRequestUpdate update) {+ if (users.size() <= 1)+ return;+CommitInfoManager commitInfoManager = GitPlex.getInstance(CommitInfoManager.class);- Map<UserFacade, Integer> contributions = new HashMap<>();+ Map<UserFacade, Long> contributions = new HashMap<>();for (UserFacade user: users)- contributions.put(user, 0);+ contributions.put(user, 0L);int count = 0;- for (String file: update.getChangedFiles()) {- int addedContributions = addContributions(contributions, commitInfoManager, file, update);+ for (FileChange change: update.getFileChanges()) {- String path = change.getNewPath();++ String path = change.getPath();+ int edits = change.getAdditions() + change.getDeletions();+ if (edits < 0)+ edits = 100;+ else if (edits == 0)+ edits = 1;+ int addedContributions = addContributions(contributions, commitInfoManager, path, edits, update);while (addedContributions == 0) {- if (file.contains("/")) {- file = StringUtils.substringBeforeLast(file, "/");- addedContributions = addContributions(contributions, commitInfoManager, file, update);+ if (path.contains("/")) {+ path = StringUtils.substringBeforeLast(path, "/");+ addedContributions = addContributions(contributions, commitInfoManager, path, edits, update);} else {- addContributions(contributions, commitInfoManager, "", update);+ addContributions(contributions, commitInfoManager, "", edits, update);break;}}@@@ -295,18 -306,28 +306,28 @@@break;}- users.sort((user1, user2)-> contributions.get(user2) - contributions.get(user1));+ Collections.sort(users, new Comparator<UserFacade>() {++ @Override+ public int compare(UserFacade o1, UserFacade o2) {+ if (contributions.get(o1) < contributions.get(o2))+ return 1;+ else+ return -1;+ }++ });}- private int addContributions(Map<UserFacade, Integer> contributions, CommitInfoManager commitInfoManager,- String file, PullRequestUpdate update) {+ private int addContributions(Map<UserFacade, Long> contributions, CommitInfoManager commitInfoManager,+ String path, int edits, PullRequestUpdate update) {int addedContributions = 0;- for (Map.Entry<UserFacade, Integer> entry: contributions.entrySet()) {+ for (Map.Entry<UserFacade, Long> entry: contributions.entrySet()) {UserFacade user = entry.getKey();- int fileContribution = commitInfoManager.getEdits(- update.getRequest().getTargetProject().getFacade(), user, file);- entry.setValue(entry.getValue() + fileContribution);- addedContributions += fileContribution;- int pathContribution = commitInfoManager.getContributions(++ int pathEdits = commitInfoManager.getEdits(+ update.getRequest().getTargetProject().getFacade(), user, path);- entry.setValue(entry.getValue() + edits*pathContribution);- addedContributions += pathContribution;++ entry.setValue(entry.getValue() + edits*pathEdits);++ addedContributions += pathEdits;}return addedContributions;}