The Unified and Autonomous Development Platform
Collect commits from master branch to provide line statistics data
Robin Shen292974c90ea6
diff --git a/core/src/main/java/com/gitplex/server/git/LineStats.java b/core/src/main/java/com/gitplex/server/git/LineStats.javaindex 00d806670b..9555180271 100644--- a/core/src/main/java/com/gitplex/server/git/LineStats.java+++ b/core/src/main/java/com/gitplex/server/git/LineStats.java@@ -1,11 +1,8 @@package com.gitplex.server.git;import java.io.Serializable;-import java.util.ArrayList;import java.util.Collections;-import java.util.Comparator;import java.util.HashMap;-import java.util.List;import java.util.Map;import javax.annotation.Nullable;@@ -16,58 +13,60 @@ public class LineStats implements Serializable {private static final long serialVersionUID = 1L;- private final Map<Day, Map<String, Integer>> dailyLines;+ private final Map<Day, Map<String, Integer>> linesByDay;- private transient List<String> sortedLanguages;+ private transient Map<String, Map<Day, Integer>> linesByLanguage;- public LineStats(Map<Day, Map<String, Integer>> dailyLines) {- this.dailyLines = dailyLines;+ public LineStats(Map<Day, Map<String, Integer>> linesByDay) {+ this.linesByDay = linesByDay;}- public Map<Day, Map<String, Integer>> getDailyLines() {- return dailyLines;+ /**+ * Get source lines by day and language+ * @return+ */+ public Map<Day, Map<String, Integer>> getLinesByDay() {+ return linesByDay;}@Nullablepublic Day getFirstDay() {- if (!dailyLines.isEmpty())- return Collections.min(dailyLines.keySet());+ if (!linesByDay.isEmpty())+ return Collections.min(linesByDay.keySet());elsereturn null;}@Nullablepublic Day getLastDay() {- if (!dailyLines.isEmpty())- return Collections.max(dailyLines.keySet());+ if (!linesByDay.isEmpty())+ return Collections.max(linesByDay.keySet());elsereturn null;}- public List<String> getSortedLanguages() {- if (sortedLanguages == null) {- Map<String, Integer> languageLines = new HashMap<>();- for (Map<String, Integer> each: dailyLines.values()) {- for (Map.Entry<String, Integer> entry: each.entrySet()) {- Integer lines = languageLines.get(entry.getKey());- if (lines != null)- lines += entry.getValue();- else- lines = entry.getValue();- languageLines.put(entry.getKey(), lines);+ /**+ * Get lines by language and day+ * @return+ */+ public Map<String, Map<Day, Integer>> getLinesByLanguage() {+ if (linesByLanguage == null) {+ linesByLanguage = new HashMap<>();+ for (Map.Entry<Day, Map<String, Integer>> outerEntry: linesByDay.entrySet()) {+ Day day = outerEntry.getKey();+ for (Map.Entry<String, Integer> innerEntry: outerEntry.getValue().entrySet()) {+ String language = innerEntry.getKey();+ int lines = innerEntry.getValue();+ Map<Day, Integer> linesOfLanguage = linesByLanguage.get(language);+ if (linesOfLanguage == null) {+ linesOfLanguage = new HashMap<>();+ linesByLanguage.put(language, linesOfLanguage);+ }+ linesOfLanguage.put(day, lines);}}- sortedLanguages = new ArrayList<>(languageLines.keySet());- Collections.sort(sortedLanguages, new Comparator<String>() {-- @Override- public int compare(String o1, String o2) {- return languageLines.get(o2) - languageLines.get(o1);- }-- });}- return sortedLanguages;+ return linesByLanguage;}}diff --git a/core/src/main/java/com/gitplex/server/git/command/LogCommit.java b/core/src/main/java/com/gitplex/server/git/command/GitCommit.javasimilarity index 77%rename from core/src/main/java/com/gitplex/server/git/command/LogCommit.javarename to core/src/main/java/com/gitplex/server/git/command/GitCommit.javaindex 6b75abe6e1..a4446e5b3b 100644--- a/core/src/main/java/com/gitplex/server/git/command/LogCommit.java+++ b/core/src/main/java/com/gitplex/server/git/command/GitCommit.java@@ -1,7 +1,6 @@package com.gitplex.server.git.command;import java.io.Serializable;-import java.util.ArrayList;import java.util.Collection;import java.util.Date;import java.util.HashSet;@@ -13,8 +12,9 @@ import org.apache.commons.lang3.StringUtils;import org.eclipse.jgit.lib.PersonIdent;import com.gitplex.server.git.GitUtils;+import com.google.common.base.Preconditions;-public class LogCommit implements Serializable {+public class GitCommit implements Serializable {private static final long serialVersionUID = 1L;@@ -30,21 +30,21 @@ public class LogCommit implements Serializable {private final List<FileChange> fileChanges;- public LogCommit(String hash, @Nullable Date commitDate, @Nullable PersonIdent committer,- @Nullable PersonIdent author, List<String> parentHashes, List<FileChange> fileChanges,- int addedLines, int deletedLines) {- this.hash = hash;+ public GitCommit(String hash, @Nullable Date commitDate, @Nullable PersonIdent committer,+ @Nullable PersonIdent author, List<String> parentHashes, @Nullable List<FileChange> fileChanges) {+ this.hash = Preconditions.checkNotNull(hash);this.commitDate = commitDate;this.committer = committer;this.author = author;- this.parentHashes = new ArrayList<>(parentHashes);- this.fileChanges = new ArrayList<>(fileChanges);+ this.parentHashes = Preconditions.checkNotNull(parentHashes);+ this.fileChanges = fileChanges;}public List<String> getParentHashes() {return parentHashes;}+ @Nullablepublic List<FileChange> getFileChanges() {return fileChanges;}@@ -74,6 +74,7 @@ public class LogCommit implements Serializable {return author;}+ @Nullablepublic Date getCommitDate() {return commitDate;}@@ -108,15 +109,11 @@ public class LogCommit implements Serializable {public Date committerDate;- public List<String> parentHashes = new ArrayList<>();+ public List<String> parentHashes;- public List<FileChange> fileChanges = new ArrayList<>();+ public List<FileChange> fileChanges;- public int addedLines;-- public int deletedLines;-- public LogCommit build() {+ public GitCommit build() {PersonIdent committer;if (StringUtils.isNotBlank(committerName) || StringUtils.isNotBlank(committerEmail))committer = GitUtils.newPersonIdent(committerName, committerEmail, committerDate);@@ -129,8 +126,7 @@ public class LogCommit implements Serializable {elseauthor = null;- return new LogCommit(hash, committerDate, committer, author, parentHashes, fileChanges,- addedLines, deletedLines);+ return new GitCommit(hash, committerDate, committer, author, parentHashes, fileChanges);}}diff --git a/core/src/main/java/com/gitplex/server/git/command/LogCommand.java b/core/src/main/java/com/gitplex/server/git/command/LogCommand.javaindex 299280125b..0ed2cc538a 100644--- a/core/src/main/java/com/gitplex/server/git/command/LogCommand.java+++ b/core/src/main/java/com/gitplex/server/git/command/LogCommand.java@@ -2,6 +2,7 @@ package com.gitplex.server.git.command;import java.io.File;import java.util.ArrayList;+import java.util.EnumSet;import java.util.List;import java.util.StringTokenizer;import java.util.concurrent.atomic.AtomicReference;@@ -20,8 +21,12 @@ public abstract class LogCommand extends GitCommand<Void> {private static final Logger logger = LoggerFactory.getLogger(LogCommand.class);+ public enum Field {PARENTS, AUTHOR, COMMITTER, FILE_CHANGES};+private List<String> revisions = new ArrayList<>();+ private EnumSet<Field> fields = EnumSet.allOf(Field.class);+public LogCommand(File gitDir) {super(gitDir);}@@ -29,35 +34,56 @@ public abstract class LogCommand extends GitCommand<Void> {public List<String> revisions() {return revisions;}-+public LogCommand revisions(List<String> revisions) {this.revisions = revisions;return this;}+ public EnumSet<Field> fields() {+ return fields;+ }++ public LogCommand fields(EnumSet<Field> fields) {+ this.fields = fields;+ return this;+ }+@Overridepublic Void call() {Preconditions.checkArgument(!revisions.isEmpty(), "Log revisions have to be specified");Commandline cmd = cmd();- String format = ""- + "hash:%H %n"- + "author:%aN %n"- + "authorEmail:%aE %n"- + "committer:%cN %n"- + "committerEmail:%cE %n"- + "parents:%P %n"- + "committerDate:%cd %n"- + "authorDate:%ad %n";+ String format = "hash:%H %n";- cmd.addArgs("-c", "diff.renameLimit=1000", "log", "--format=" + format, "--date=raw", "--numstat",- "--find-renames");+ if (fields.contains(Field.PARENTS)) {+ format += "parents:%P %n";+ }+ if (fields.contains(Field.AUTHOR)) {+ format += ""+ + "author:%aN %n"+ + "authorEmail:%aE %n"+ + "authorDate:%ad %n";+ }+ if (fields.contains(Field.COMMITTER)) {+ format += ""+ + "committer:%cN %n"+ + "committerEmail:%cE %n"+ + "committerDate:%cd %n";+ }++ if (fields.contains(Field.FILE_CHANGES))+ cmd.addArgs("-c", "diff.renameLimit=1000", "log", "--numstat", "--find-renames");+ else+ cmd.addArgs("log");++ cmd.addArgs("--format=" + format, "--date=raw");for (String revision: revisions)cmd.addArgs(revision);- AtomicReference<LogCommit.Builder> commitBuilderRef = new AtomicReference<>();+ AtomicReference<GitCommit.Builder> commitBuilderRef = new AtomicReference<>(null);cmd.execute(new LineConsumer() {@Override@@ -65,7 +91,11 @@ public abstract class LogCommand extends GitCommand<Void> {if (line.startsWith("hash:")) {if (commitBuilderRef.get() != null)LogCommand.this.consume(commitBuilderRef.get().build());- commitBuilderRef.set(new LogCommit.Builder());+ commitBuilderRef.set(new GitCommit.Builder());+ if (fields.contains(Field.PARENTS))+ commitBuilderRef.get().parentHashes = new ArrayList<>();+ if (fields.contains(Field.FILE_CHANGES))+ commitBuilderRef.get().fileChanges = new ArrayList<>();commitBuilderRef.get().hash = line.substring("hash:".length()).trim();} else if (line.startsWith("author:")) {commitBuilderRef.get().authorName = line.substring("author:".length()).trim();@@ -135,6 +165,6 @@ public abstract class LogCommand extends GitCommand<Void> {return null;}- protected abstract void consume(LogCommit commit);+ protected abstract void consume(GitCommit commit);}diff --git a/core/src/main/java/com/gitplex/server/git/command/RevListCommand.java b/core/src/main/java/com/gitplex/server/git/command/RevListCommand.javaindex 33c02ddafd..578a5e0598 100644--- a/core/src/main/java/com/gitplex/server/git/command/RevListCommand.java+++ b/core/src/main/java/com/gitplex/server/git/command/RevListCommand.java@@ -15,6 +15,8 @@ import com.gitplex.utils.command.LineConsumer;public class RevListCommand extends GitCommand<List<String>> {private static final Logger logger = LoggerFactory.getLogger(RevListCommand.class);++ public enum Order {DATE, AUTHOR_DATE, TOPO};private List<String> revisions = new ArrayList<>();@@ -28,6 +30,10 @@ public class RevListCommand extends GitCommand<List<String>> {private int skip;+ private Order order;++ private boolean firstParent;+private boolean ignoreCase;private List<String> messages = new ArrayList<>();@@ -67,6 +73,24 @@ public class RevListCommand extends GitCommand<List<String>> {return this;}+ public Order order() {+ return order;+ }++ public RevListCommand order(Order order) {+ this.order = order;+ return this;+ }++ public boolean firstParent() {+ return firstParent;+ }++ public RevListCommand firstParent(boolean firstParent) {+ this.firstParent = firstParent;+ return this;+ }+public RevListCommand after(Date after) {this.after = DateFormatUtils.ISO_DATE_FORMAT.format(after);return this;@@ -166,7 +190,17 @@ public class RevListCommand extends GitCommand<List<String>> {cmd.addArgs("-" + count);if (skip != 0)cmd.addArgs("--skip=" + skip);-++ if (order == Order.DATE)+ cmd.addArgs("--date-order");+ else if (order == Order.AUTHOR_DATE)+ cmd.addArgs("--author-date-order");+ else if (order == Order.TOPO)+ cmd.addArgs("--topo-order");++ if (firstParent)+ cmd.addArgs("--first-parent");+for (String author: authors)cmd.addArgs("--author=" + author);diff --git a/core/src/main/java/com/gitplex/server/manager/CommitInfoManager.java b/core/src/main/java/com/gitplex/server/manager/CommitInfoManager.javaindex 3f0d3fe72a..494a63c4ea 100644--- a/core/src/main/java/com/gitplex/server/manager/CommitInfoManager.java+++ b/core/src/main/java/com/gitplex/server/manager/CommitInfoManager.java@@ -9,6 +9,7 @@ import org.eclipse.jgit.lib.ObjectId;import com.gitplex.server.git.Contribution;import com.gitplex.server.git.Contributor;+import com.gitplex.server.git.LineStats;import com.gitplex.server.git.NameAndEmail;import com.gitplex.server.model.Project;import com.gitplex.server.util.Day;@@ -83,5 +84,15 @@ public interface CommitInfoManager {*/List<Contributor> getTopContributors(Project project, int top, Contribution.Type orderBy,Day fromDay, Day toDay);++ /**+ * Get source code line statistics over time+ *+ * @param project+ * project to get line stats for+ * @return+ * line statistics data+ */+ LineStats getLineStats(Project project);}\ No newline at end of filediff --git a/core/src/main/java/com/gitplex/server/manager/impl/DefaultCommitInfoManager.java b/core/src/main/java/com/gitplex/server/manager/impl/DefaultCommitInfoManager.javaindex 8846105e44..19d39fc9ad 100644--- a/core/src/main/java/com/gitplex/server/manager/impl/DefaultCommitInfoManager.java+++ b/core/src/main/java/com/gitplex/server/manager/impl/DefaultCommitInfoManager.java@@ -41,11 +41,13 @@ import com.gitplex.server.event.lifecycle.SystemStarted;import com.gitplex.server.git.Contribution;import com.gitplex.server.git.Contributor;import com.gitplex.server.git.GitUtils;+import com.gitplex.server.git.LineStats;import com.gitplex.server.git.NameAndEmail;import com.gitplex.server.git.command.FileChange;+import com.gitplex.server.git.command.GitCommit;import com.gitplex.server.git.command.LogCommand;-import com.gitplex.server.git.command.LogCommit;import com.gitplex.server.git.command.RevListCommand;+import com.gitplex.server.git.command.RevListCommand.Order;import com.gitplex.server.manager.BatchWorkManager;import com.gitplex.server.manager.CommitInfoManager;import com.gitplex.server.manager.ProjectManager;@@ -53,7 +55,6 @@ import com.gitplex.server.manager.StorageManager;import com.gitplex.server.model.Project;import com.gitplex.server.persistence.UnitOfWork;import com.gitplex.server.persistence.annotation.Sessional;-import com.gitplex.server.persistence.dao.Dao;import com.gitplex.server.persistence.dao.EntityRemoved;import com.gitplex.server.util.BatchWorker;import com.gitplex.server.util.Day;@@ -86,7 +87,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager implemeprivate static final long LOG_FILE_SIZE = 256*1024;- private static final int COLLECT_BATCH_SIZE = 50000;+ private static final int COLLECT_BATCH_SIZE = 10000;private static final int MAX_COLLECTING_FILES = 50000;@@ -118,6 +119,10 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager implemeprivate static final ByteIterable LAST_COMMIT_KEY = new StringByteIterable("lastCommit");+ private static final ByteIterable LAST_COMMIT_OF_DEFAULT_BRANCH_KEY = new StringByteIterable("lastCommitOfDefaultBranch");++ private static final ByteIterable LINE_STATS_KEY = new StringByteIterable("lineStats");+private static final ByteIterable USERS_KEY = new StringByteIterable("users");private static final ByteIterable FILES_KEY = new StringByteIterable("files");@@ -128,16 +133,148 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager implemeprivate static final int PRIORITY = 100;+ private static final Map<String, String> PROGRAMMING_LANGUAGES = new HashMap<>();++ static {+ PROGRAMMING_LANGUAGES.put("java", "Java");++ PROGRAMMING_LANGUAGES.put("h", "C");+ PROGRAMMING_LANGUAGES.put("c", "C");++ PROGRAMMING_LANGUAGES.put("h++", "C++");+ PROGRAMMING_LANGUAGES.put("c++", "C++");+ PROGRAMMING_LANGUAGES.put("hpp", "C++");+ PROGRAMMING_LANGUAGES.put("cpp", "C++");+ PROGRAMMING_LANGUAGES.put("hxx", "C++");+ PROGRAMMING_LANGUAGES.put("cxx", "C++");+ PROGRAMMING_LANGUAGES.put("hh", "C++");+ PROGRAMMING_LANGUAGES.put("cc", "C++");++ PROGRAMMING_LANGUAGES.put("cob", "Cobol");+ PROGRAMMING_LANGUAGES.put("cpy", "Cobol");++ PROGRAMMING_LANGUAGES.put("cs", "CSharp");++ PROGRAMMING_LANGUAGES.put("clj", "Clojure");+ PROGRAMMING_LANGUAGES.put("cljc", "Clojure");+ PROGRAMMING_LANGUAGES.put("cljx", "Clojure");+ PROGRAMMING_LANGUAGES.put("cljs", "ClojureScript");+ PROGRAMMING_LANGUAGES.put("gss", "Closure Stylesheets");+ PROGRAMMING_LANGUAGES.put("coffee", "CoffeeScript");++ PROGRAMMING_LANGUAGES.put("cl", "Common Lisp");+ PROGRAMMING_LANGUAGES.put("lisp", "Common Lisp");+ PROGRAMMING_LANGUAGES.put("el", "Common Lisp");++ PROGRAMMING_LANGUAGES.put("css", "CSS");+ PROGRAMMING_LANGUAGES.put("d", "D");+ PROGRAMMING_LANGUAGES.put("dart", "Dart");+ PROGRAMMING_LANGUAGES.put("dtd", "DTD");++ PROGRAMMING_LANGUAGES.put("erl", "Erlang");+ PROGRAMMING_LANGUAGES.put("f", "Fortran");+ PROGRAMMING_LANGUAGES.put("for", "Fortran");+ PROGRAMMING_LANGUAGES.put("f77", "Fortran");+ PROGRAMMING_LANGUAGES.put("f90", "Fortran");++ PROGRAMMING_LANGUAGES.put("go", "Go");+ PROGRAMMING_LANGUAGES.put("groovy", "Groovy");+ PROGRAMMING_LANGUAGES.put("gradle", "Groovy");++ PROGRAMMING_LANGUAGES.put("hs", "Haskell");+ PROGRAMMING_LANGUAGES.put("aspx", "ASP.NET");+ PROGRAMMING_LANGUAGES.put("html", "HTML");+ PROGRAMMING_LANGUAGES.put("htm", "HTML");+ PROGRAMMING_LANGUAGES.put("jsp", "Java Server Pages");+ PROGRAMMING_LANGUAGES.put("js", "JavaScript");++ PROGRAMMING_LANGUAGES.put("json", "JSON");+ PROGRAMMING_LANGUAGES.put("jsx", "JSX");++ PROGRAMMING_LANGUAGES.put("kt", "Kotlin");+ PROGRAMMING_LANGUAGES.put("less", "LESS");+ PROGRAMMING_LANGUAGES.put("lua", "Lua");+ PROGRAMMING_LANGUAGES.put("md", "Markdown");+ PROGRAMMING_LANGUAGES.put("mkd", "Markdown");+ PROGRAMMING_LANGUAGES.put("m", "Objective-C");+ PROGRAMMING_LANGUAGES.put("mm", "Objective-C");++ PROGRAMMING_LANGUAGES.put("p", "Pascal");+ PROGRAMMING_LANGUAGES.put("pas", "Pascal");+ PROGRAMMING_LANGUAGES.put("pl", "Perl");+ PROGRAMMING_LANGUAGES.put("pm", "Perl");++ PROGRAMMING_LANGUAGES.put("php", "PHP");+ PROGRAMMING_LANGUAGES.put("php3", "PHP");+ PROGRAMMING_LANGUAGES.put("php4", "PHP");+ PROGRAMMING_LANGUAGES.put("php5", "PHP");+ PROGRAMMING_LANGUAGES.put("php7", "PHP");+ PROGRAMMING_LANGUAGES.put("phtml", "PHP");++ PROGRAMMING_LANGUAGES.put("sql", "SQL");+ PROGRAMMING_LANGUAGES.put("ps1", "PowerShell");+ PROGRAMMING_LANGUAGES.put("psd1", "PowerShell");+ PROGRAMMING_LANGUAGES.put("psm1", "PowerShell");+ PROGRAMMING_LANGUAGES.put("psm1", "PowerShell");++ PROGRAMMING_LANGUAGES.put("properties", "Properties");+ PROGRAMMING_LANGUAGES.put("ini", "INI");+ PROGRAMMING_LANGUAGES.put("in", "INI");+ PROGRAMMING_LANGUAGES.put("proto", "ProtoBuf");++ PROGRAMMING_LANGUAGES.put("BUILD", "Python");+ PROGRAMMING_LANGUAGES.put("py", "Python");+ PROGRAMMING_LANGUAGES.put("pyw", "Python");+ PROGRAMMING_LANGUAGES.put("bzl", "Python");++ PROGRAMMING_LANGUAGES.put("pp", "Puppet");+ PROGRAMMING_LANGUAGES.put("r", "R");+ PROGRAMMING_LANGUAGES.put("R", "R");+ PROGRAMMING_LANGUAGES.put("rb", "Ruby");+ PROGRAMMING_LANGUAGES.put("rs", "Rust");++ PROGRAMMING_LANGUAGES.put("sas", "SAS");+ PROGRAMMING_LANGUAGES.put("sass", "Sass");+ PROGRAMMING_LANGUAGES.put("scala", "Scala");+ PROGRAMMING_LANGUAGES.put("scm", "Scheme");+ PROGRAMMING_LANGUAGES.put("ss", "Scheme");+ PROGRAMMING_LANGUAGES.put("scss", "Scss");++ PROGRAMMING_LANGUAGES.put("sh", "Shell");+ PROGRAMMING_LANGUAGES.put("ksh", "Shell");+ PROGRAMMING_LANGUAGES.put("bash", "Shell");++ PROGRAMMING_LANGUAGES.put("st", "SmallTalk");+ PROGRAMMING_LANGUAGES.put("soy", "Soy");+ PROGRAMMING_LANGUAGES.put("styl", "Stylus");+ PROGRAMMING_LANGUAGES.put("swift", "Swift");++ PROGRAMMING_LANGUAGES.put("tex", "LaTeX");+ PROGRAMMING_LANGUAGES.put("tcl", "TCL");+ PROGRAMMING_LANGUAGES.put("ts", "TypeScript");+ PROGRAMMING_LANGUAGES.put("tsx", "TypeScript-JSX");+ PROGRAMMING_LANGUAGES.put("vb", "VB.NET");+ PROGRAMMING_LANGUAGES.put("vbs", "VBScript");++ PROGRAMMING_LANGUAGES.put("vue", "Vue.js Component");+ PROGRAMMING_LANGUAGES.put("xml", "XML");+ PROGRAMMING_LANGUAGES.put("xsl", "XML");+ PROGRAMMING_LANGUAGES.put("xsd", "XML");+ PROGRAMMING_LANGUAGES.put("svg", "XML");+ PROGRAMMING_LANGUAGES.put("yaml", "Yaml");+ PROGRAMMING_LANGUAGES.put("yml", "Yaml");+ }+private final StorageManager storageManager;private final BatchWorkManager batchWorkManager;- private final ExecutorService executorService;-private final ProjectManager projectManager;private final UnitOfWork unitOfWork;+ private final ExecutorService executorService;+private final Map<Long, List<String>> filesCache = new ConcurrentHashMap<>();private final Map<Long, Integer> commitCountCache = new ConcurrentHashMap<>();@@ -146,7 +283,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme@Injectpublic DefaultCommitInfoManager(ProjectManager projectManager, StorageManager storageManager,- BatchWorkManager batchWorkManager, UnitOfWork unitOfWork, Dao dao, ExecutorService executorService) {+ BatchWorkManager batchWorkManager, UnitOfWork unitOfWork, ExecutorService executorService) {this.projectManager = projectManager;this.storageManager = storageManager;this.batchWorkManager = batchWorkManager;@@ -154,328 +291,445 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager implemethis.executorService = executorService;}- private void doCollect(Project project, ObjectId commitId, boolean divide) {+ private boolean isCommitCollected(byte[] commitBytes) {+ /*+ * Collected commits stores an additional byte to differentiate from those not collected but with parent+ * information stored+ */+ return commitBytes != null && commitBytes.length % 20 != 0;+ }++ private void doCollect(Project project, ObjectId commitId, String refName) {Environment env = getEnv(project.getId().toString());Store defaultStore = getStore(env, DEFAULT_STORE);-+ Store commitsStore = getStore(env, COMMITS_STORE);+ Store editsStore = getStore(env, EDITS_STORE);+ Store historyPathsStore = getStore(env, HISTORY_PATHS_STORE);+ Store pathToIndexStore = getStore(env, PATH_TO_INDEX_STORE);+ Store indexToPathStore = getStore(env, INDEX_TO_PATH_STORE);+ Store emailToIndexStore = getStore(env, EMAIL_TO_INDEX_STORE);+ Store indexToUserStore = getStore(env, INDEX_TO_USER_STORE);+ Store dailyContributionsStore = getStore(env, DAILY_CONTRIBUTIONS_STORE);+Repository repository = project.getRepository();- if (divide) {- List<ObjectId> intermediateCommitIds = new ArrayList<>();- List<String> revisions = new ArrayList<>();- revisions.add(commitId.name());- ObjectId lastCommitId = env.computeInTransaction(new TransactionalComputable<ObjectId>() {+ env.executeInTransaction(new TransactionalExecutable() {++ @Override+ public void execute(Transaction txn) {+ byte[] commitKeyBytes = new byte[20];+ commitId.copyRawTo(commitKeyBytes, 0);+ ByteIterable commitKey = new ArrayByteIterable(commitKeyBytes);+ byte[] commitBytes = readBytes(commitsStore, txn, commitKey);++ if (!isCommitCollected(commitBytes)) {+ int commitCount = readInt(defaultStore, txn, COMMIT_COUNT_KEY, 0);++ try (RevWalk revWalk = new RevWalk(project.getRepository())) {+ RevCommit commit = revWalk.lookupCommit(commitId);+ revWalk.markStart(commit);++ byte[] lastCommitBytes = readBytes(defaultStore, txn, LAST_COMMIT_KEY);+ if (lastCommitBytes != null) {+ ObjectId lastCommitId = ObjectId.fromRaw(lastCommitBytes);+ if (repository.hasObject(lastCommitId))+ revWalk.markUninteresting(revWalk.lookupCommit(lastCommitId));+ }++ RevCommit nextCommit = revWalk.next();+ while (nextCommit != null) {+ byte[] nextCommitKeyBytes = new byte[20];+ nextCommit.copyRawTo(nextCommitKeyBytes, 0);+ ByteIterable nextCommitKey = new ArrayByteIterable(nextCommitKeyBytes);+ byte[] nextCommitBytes = readBytes(commitsStore, txn, nextCommitKey);++ if (!isCommitCollected(nextCommitBytes)) {+ byte[] newNextCommitBytes;+ if (nextCommitBytes == null) {+ newNextCommitBytes = new byte[1];+ } else {+ newNextCommitBytes = new byte[1+nextCommitBytes.length];+ System.arraycopy(nextCommitBytes, 0, newNextCommitBytes, 1, nextCommitBytes.length);+ }++ commitsStore.put(txn, nextCommitKey, new ArrayByteIterable(newNextCommitBytes));++ commitCount++;++ for (RevCommit parentCommit: nextCommit.getParents()) {+ byte[] parentCommitKeyBytes = new byte[20];+ parentCommit.copyRawTo(parentCommitKeyBytes, 0);+ ByteIterable parentCommitKey = new ArrayByteIterable(parentCommitKeyBytes);+ byte[] parentCommitBytes = readBytes(commitsStore, txn, parentCommitKey);+ byte[] newParentCommitBytes;+ if (parentCommitBytes != null) {+ newParentCommitBytes = new byte[parentCommitBytes.length+20];+ System.arraycopy(parentCommitBytes, 0, newParentCommitBytes, 0, parentCommitBytes.length);+ } else {+ newParentCommitBytes = new byte[20];+ }+ nextCommit.copyRawTo(newParentCommitBytes, newParentCommitBytes.length-20);+ commitsStore.put(txn, parentCommitKey, new ArrayByteIterable(newParentCommitBytes));+ }++ }+ nextCommit = revWalk.next();+ }+ writeInt(defaultStore, txn, COMMIT_COUNT_KEY, commitCount);+ commitCountCache.remove(project.getId());++ lastCommitBytes = new byte[20];+ commitId.copyRawTo(lastCommitBytes, 0);+ defaultStore.put(txn, LAST_COMMIT_KEY, new ArrayByteIterable(lastCommitBytes));+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ }+ }++ });++ if (GitUtils.branch2ref(project.getDefaultBranch()).equals(refName)) {+ ObjectId lastCommitId =+ env.computeInReadonlyTransaction(new TransactionalComputable<ObjectId>() {@Overridepublic ObjectId compute(Transaction txn) {- byte[] bytes = readBytes(defaultStore, txn, LAST_COMMIT_KEY);- return bytes!=null? ObjectId.fromRaw(bytes): null;+ byte[] lastCommitBytes = readBytes(defaultStore, txn, LAST_COMMIT_OF_DEFAULT_BRANCH_KEY);+ if (lastCommitBytes != null)+ return ObjectId.fromRaw(lastCommitBytes);+ else+ return null;}});- if (lastCommitId != null) {- if (repository.hasObject(lastCommitId))++ if (!commitId.equals(lastCommitId)) {+ RevListCommand revList = new RevListCommand(project.getGitDir());+ List<String> revisions = new ArrayList<>();+ revisions.add(commitId.name());+ if (lastCommitId != null && repository.hasObject(lastCommitId))revisions.add("^" + lastCommitId.name());- }-- int count = 0;- for (String commitHash: new RevListCommand(repository.getDirectory()).revisions(revisions).call()) {- count++;- if (count > COLLECT_BATCH_SIZE) {- intermediateCommitIds.add(ObjectId.fromString(commitHash));- count = 0;+ revList.revisions(revisions).order(Order.TOPO);++ List<ObjectId> historyIds = new ArrayList<>();+ for (String commitHash: revList.call())+ historyIds.add(ObjectId.fromString(commitHash));++ revList = new RevListCommand(project.getGitDir());+ revList.order(null).firstParent(true);++ Set<ObjectId> firstParentIds = new HashSet<>();+ for (String commitHash: revList.call())+ firstParentIds.add(ObjectId.fromString(commitHash));++ /*+ * Instead of collecting information of master branch all at once, we identify some+ * intermediate commits and collect the information using these intermediate commits+ * multiple times for two reasons:+ * 1. Use less memory+ * 2. Commit Exodus transaction sooner so user can use auto-completion when search+ * commits even if collection is not done yet+ */+ List<ObjectId> intermediateIds = new ArrayList<>();+ int count = 0;+ for (ObjectId historyId: historyIds) {+ count++;+ /*+ * Only use intermediate commits that are part of first parent chain. This+ * makes sure that subsequent intermediate commits are always ancestor of+ * current intermediate commit (after reverse done below), to avoid+ * collecting some commits multiple times+ */+ if (count > COLLECT_BATCH_SIZE && firstParentIds.contains(historyId)) {+ intermediateIds.add(historyId);+ count = 0;+ }}- }-- for (int i=intermediateCommitIds.size()-1; i>=0; i--) {- doCollect(project, intermediateCommitIds.get(i), false);- }-- doCollect(project, commitId, false);- } else {- Store commitsStore = getStore(env, COMMITS_STORE);- Store editsStore = getStore(env, EDITS_STORE);- Store historyPathsStore = getStore(env, HISTORY_PATHS_STORE);- Store pathToIndexStore = getStore(env, PATH_TO_INDEX_STORE);- Store indexToPathStore = getStore(env, INDEX_TO_PATH_STORE);- Store emailToIndexStore = getStore(env, EMAIL_TO_INDEX_STORE);- Store indexToUserStore = getStore(env, INDEX_TO_USER_STORE);- Store dailyContributionsStore = getStore(env, DAILY_CONTRIBUTIONS_STORE);-- env.executeInTransaction(new TransactionalExecutable() {++ Collections.reverse(intermediateIds);+ intermediateIds.add(commitId);- @SuppressWarnings("unchecked")- @Override- public void execute(Transaction txn) {- try {- byte[] bytes = readBytes(defaultStore, txn, LAST_COMMIT_KEY);+ historyIds = null;+ firstParentIds = null;++ for(ObjectId currentCommitId: intermediateIds) {+ env.executeInTransaction(new TransactionalExecutable() {- ObjectId lastCommitId;- if (bytes != null) {- ObjectId commitId = ObjectId.fromRaw(bytes);- if (repository.hasObject(commitId))- lastCommitId = commitId;+ @SuppressWarnings("unchecked")+ @Override+ public void execute(Transaction txn) {+ NextIndex nextIndex = new NextIndex();+ nextIndex.user = readInt(defaultStore, txn, NEXT_USER_INDEX_KEY, 0);+ nextIndex.path = readInt(defaultStore, txn, NEXT_PATH_INDEX_KEY, 0);++ Map<Long, Integer> editsCache = new HashMap<>();++ Set<NameAndEmail> users;+ byte[] userBytes = readBytes(defaultStore, txn, USERS_KEY);+ if (userBytes != null)+ users = (Set<NameAndEmail>) SerializationUtils.deserialize(userBytes);else- lastCommitId = null;- } else {- lastCommitId = null;- }-- int commitCount = readInt(defaultStore, txn, COMMIT_COUNT_KEY, 0);-- NextIndex nextIndex = new NextIndex();- nextIndex.user = readInt(defaultStore, txn, NEXT_USER_INDEX_KEY, 0);- nextIndex.path = readInt(defaultStore, txn, NEXT_PATH_INDEX_KEY, 0);-- Map<Integer, Map<Integer, Contribution>> dailyContributionsCache = new HashMap<>();- Map<Integer, Contribution> overallContributions =- deserializeContributions(readBytes(defaultStore, txn, OVERALL_CONTRIBUTIONS_KEY));- Map<Long, Integer> editsCache = new HashMap<>();-- Set<NameAndEmail> users;- bytes = readBytes(defaultStore, txn, USERS_KEY);- if (bytes != null)- users = (Set<NameAndEmail>) SerializationUtils.deserialize(bytes);- else- users = new HashSet<>();+ users = new HashSet<>();- Map<String, Long> files;- bytes = readBytes(defaultStore, txn, FILES_KEY);- if (bytes != null)- files = (Map<String, Long>) SerializationUtils.deserialize(bytes);- else- files = new HashMap<>();-- /*- * Use a synchronous queue to achieve below purpose:- * 1. Add commit to Xodus transactional store in the same thread opening the transaction- * as this is required by Xodus- * 2. Do not pile up commits to use minimal memory- */- SynchronousQueue<Optional<LogCommit>> queue = new SynchronousQueue<>();- AtomicReference<Exception> logException = new AtomicReference<>(null);- executorService.execute(new Runnable() {+ Map<String, Long> files;+ byte[] fileBytes = readBytes(defaultStore, txn, FILES_KEY);+ if (fileBytes != null)+ files = (Map<String, Long>) SerializationUtils.deserialize(fileBytes);+ else+ files = new HashMap<>();++ Map<Integer, Map<String, Integer>> lineStats;+ byte[] bytesOfLineStats = readBytes(defaultStore, txn, LINE_STATS_KEY);+ if (bytesOfLineStats != null) {+ lineStats = (Map<Integer, Map<String, Integer>>) SerializationUtils.deserialize(+ bytesOfLineStats);+ } else {+ lineStats = new HashMap<>();+ }++ Map<Integer, Map<Integer, Contribution>> dailyContributionsCache = new HashMap<>();+ Map<Integer, Contribution> overallContributions =+ deserializeContributions(readBytes(defaultStore, txn, OVERALL_CONTRIBUTIONS_KEY));- @Override- public void run() {- try {- List<String> revisions = new ArrayList<>();- revisions.add(commitId.name());- if (lastCommitId != null)- revisions.add("^" + lastCommitId.name());-- new LogCommand(repository.getDirectory()) {+ /*+ * Use a synchronous queue to achieve below purpose:+ * 1. Add commit to Xodus transactional store in the same thread opening the transaction+ * as this is required by Xodus+ * 2. Do not pile up commits to use minimal memory+ */+ SynchronousQueue<Optional<GitCommit>> queue = new SynchronousQueue<>();+ AtomicReference<Exception> logException = new AtomicReference<>(null);++ List<String> revisions = new ArrayList<>();+ revisions.add(currentCommitId.name());- @Override- protected void consume(LogCommit commit) {- try {- queue.put(Optional.of(commit));- } catch (InterruptedException e) {- }- }-- }.revisions(revisions).call();-- } catch (Exception e) {- logException.set(e);- } finally {+ ObjectId lastCommitId;+ byte[] lastCommitBytes = readBytes(defaultStore, txn, LAST_COMMIT_OF_DEFAULT_BRANCH_KEY);+ if (lastCommitBytes != null)+ lastCommitId = ObjectId.fromRaw(lastCommitBytes);+ else+ lastCommitId = null;+ if (lastCommitId != null && repository.hasObject(lastCommitId))+ revisions.add("^" + lastCommitId.name());++ LogCommand log = new LogCommand(project.getGitDir()) {++ @Override+ protected void consume(GitCommit commit) {try {- queue.put(Optional.empty());+ queue.put(Optional.of(commit));} catch (InterruptedException e) {}}- }-- });-- Optional<LogCommit> commitOptional = queue.take();- while (commitOptional.isPresent()) {- LogCommit commit = commitOptional.get();-- byte[] keyBytes = new byte[20];- ObjectId.fromString(commit.getHash()).copyRawTo(keyBytes, 0);- ByteIterable key = new ArrayByteIterable(keyBytes);- byte[] valueBytes = readBytes(commitsStore, txn, key);-- if (valueBytes == null || valueBytes.length % 20 == 0) {- /*- * Length of stored bytes of a commit is either 20*nChild- * (20 is length of ObjectId), or 1+20*nChild, as we need- * an extra leading byte to differentiate commits being- * processed and commits with child information attached- * but not processed.- */- byte[] newValueBytes;- if (valueBytes == null) {- newValueBytes = new byte[1];- } else {- newValueBytes = new byte[valueBytes.length+1];- System.arraycopy(valueBytes, 0, newValueBytes, 1, valueBytes.length);- }- commitsStore.put(txn, key, new ArrayByteIterable(newValueBytes));-- commitCount++;- for (String parentHash: commit.getParentHashes()) {- keyBytes = new byte[20];- ObjectId.fromString(parentHash).copyRawTo(keyBytes, 0);- key = new ArrayByteIterable(keyBytes);- valueBytes = readBytes(commitsStore, txn, key);- if (valueBytes != null) {- newValueBytes = new byte[valueBytes.length+20];- System.arraycopy(valueBytes, 0, newValueBytes, 0, valueBytes.length);- } else {- newValueBytes = new byte[20];+ }.revisions(revisions);++ executorService.execute(new Runnable() {++ @Override+ public void run() {+ try {+ log.call();+ } catch (Exception e) {+ logException.set(e);+ } finally {+ try {+ queue.put(Optional.empty());+ } catch (InterruptedException e) {+ }}- ObjectId.fromString(commit.getHash()).copyRawTo(newValueBytes, newValueBytes.length-20);- commitsStore.put(txn, key, new ArrayByteIterable(newValueBytes));- }-- if (commit.getCommitDate() != null) {- for (String file: commit.getChangedFiles())- files.put(file, commit.getCommitDate().getTime());}- if (commit.getCommitter() != null)- users.add(new NameAndEmail(commit.getCommitter()));-- if (commit.getAuthor() != null) {- NameAndEmail nameAndEmail = new NameAndEmail(commit.getAuthor());- users.add(nameAndEmail);+ });++ try {+ Optional<GitCommit> logCommitOptional = queue.take();+ while (logCommitOptional.isPresent()) {+ GitCommit logCommit = logCommitOptional.get();- String emailAddress = commit.getAuthor().getEmailAddress();- if (StringUtils.isNotBlank(emailAddress)) {- key = new StringByteIterable(emailAddress);- int userIndex = readInt(emailToIndexStore, txn, key, -1);- if (userIndex == -1) {- userIndex = nextIndex.user++;- writeInt(emailToIndexStore, txn, key, userIndex);- indexToUserStore.put(txn,- new IntByteIterable(userIndex),- new ArrayByteIterable(SerializationUtils.serialize(nameAndEmail)));- }+ if (logCommit.getCommitDate() != null) {+ for (String file: logCommit.getChangedFiles())+ files.put(file, logCommit.getCommitDate().getTime());+ }++ if (logCommit.getCommitter() != null)+ users.add(new NameAndEmail(logCommit.getCommitter()));++ if (logCommit.getAuthor() != null) {+ NameAndEmail nameAndEmail = new NameAndEmail(logCommit.getAuthor());+ users.add(nameAndEmail);- for (FileChange change: commit.getFileChanges()) {- String path = change.getPath();- int pathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,- nextIndex, path);- int edits = change.getAdditions() + change.getDeletions();- updateEdits(editsStore, txn, editsCache,- userIndex, pathIndex, edits);- while (path.contains("/")) {- path = StringUtils.substringBeforeLast(path, "/");- pathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,- nextIndex, path);- updateEdits(editsStore, txn, editsCache,- userIndex, pathIndex, edits);+ String emailAddress = logCommit.getAuthor().getEmailAddress();+ if (StringUtils.isNotBlank(emailAddress)) {+ ByteIterable emailKey = new StringByteIterable(emailAddress);+ int userIndex = readInt(emailToIndexStore, txn, emailKey, -1);+ if (userIndex == -1) {+ userIndex = nextIndex.user++;+ writeInt(emailToIndexStore, txn, emailKey, userIndex);+ indexToUserStore.put(txn,+ new IntByteIterable(userIndex),+ new ArrayByteIterable(SerializationUtils.serialize(nameAndEmail)));}- pathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,- nextIndex, "");- updateEdits(editsStore, txn, editsCache, userIndex, pathIndex, edits);- }-- if (commit.getCommitDate() != null && commit.getParentHashes().size() <= 1) {- int dayValue = new Day(commit.getCommitDate()).getValue();- Map<Integer, Contribution> contributionsOnDay =- dailyContributionsCache.get(dayValue);- if (contributionsOnDay == null) {- contributionsOnDay = deserializeContributions(readBytes(- dailyContributionsStore, txn, new IntByteIterable(dayValue)));- dailyContributionsCache.put(dayValue, contributionsOnDay);++ if (logCommit.getCommitDate() != null && logCommit.getParentHashes().size() <= 1) {+ int dayValue = new Day(logCommit.getCommitDate()).getValue();+ Map<Integer, Contribution> contributionsOnDay =+ dailyContributionsCache.get(dayValue);+ if (contributionsOnDay == null) {+ contributionsOnDay = deserializeContributions(readBytes(+ dailyContributionsStore, txn, new IntByteIterable(dayValue)));+ dailyContributionsCache.put(dayValue, contributionsOnDay);+ }+ updateContribution(contributionsOnDay, userIndex, logCommit);}- updateContribution(contributionsOnDay, userIndex, commit);- updateContribution(overallContributions, dayValue, commit);+ for (FileChange change: logCommit.getFileChanges()) {+ String path = change.getPath();+ int pathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,+ nextIndex, path);+ int edits = change.getAdditions() + change.getDeletions();+ updateEdits(editsStore, txn, editsCache, userIndex, pathIndex, edits);+ while (path.contains("/")) {+ path = StringUtils.substringBeforeLast(path, "/");+ pathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,+ nextIndex, path);+ updateEdits(editsStore, txn, editsCache, userIndex, pathIndex, edits);+ }+ pathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,+ nextIndex, "");+ updateEdits(editsStore, txn, editsCache, userIndex, pathIndex, edits);+ }}}- }-- for (FileChange change: commit.getFileChanges()) {- if (change.getOldPath() != null) {- int pathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,- nextIndex, change.getPath());- ByteIterable pathKey = new IntByteIterable(pathIndex);- Set<Integer> historyPathIndexes = new HashSet<>();- byte[] bytesOfHistoryPaths = readBytes(historyPathsStore, txn, pathKey);- if (bytesOfHistoryPaths == null) {- bytesOfHistoryPaths = new byte[0];- int pos = 0;- for (int i=0; i<bytesOfHistoryPaths.length/Integer.SIZE; i++) {- historyPathIndexes.add(ByteBuffer.wrap(bytesOfHistoryPaths, pos, Integer.SIZE).getInt());- pos += Integer.SIZE;++ if (logCommit.getCommitDate() != null && logCommit.getParentHashes().size() <= 1) {+ int dayValue = new Day(logCommit.getCommitDate()).getValue();+ updateContribution(overallContributions, dayValue, logCommit);+ }++ for (FileChange change: logCommit.getFileChanges()) {+ if (change.getOldPath() != null) {+ int pathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,+ nextIndex, change.getPath());+ ByteIterable pathKey = new IntByteIterable(pathIndex);+ Set<Integer> historyPathIndexes = new HashSet<>();+ byte[] bytesOfHistoryPaths = readBytes(historyPathsStore, txn, pathKey);+ if (bytesOfHistoryPaths == null) {+ bytesOfHistoryPaths = new byte[0];+ int pos = 0;+ for (int i=0; i<bytesOfHistoryPaths.length/Integer.SIZE; i++) {+ historyPathIndexes.add(ByteBuffer.wrap(bytesOfHistoryPaths, pos, Integer.SIZE).getInt());+ pos += Integer.SIZE;+ }+ } else {+ historyPathIndexes = new HashSet<>();+ }+ if (historyPathIndexes.size() < MAX_HISTORY_PATHS) {+ int oldPathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,+ nextIndex, change.getOldPath());+ if (!historyPathIndexes.contains(oldPathIndex)) {+ historyPathIndexes.add(oldPathIndex);+ byte[] newBytesOfHistoryPaths =+ new byte[bytesOfHistoryPaths.length+Integer.SIZE];+ System.arraycopy(bytesOfHistoryPaths, 0,+ newBytesOfHistoryPaths, 0, bytesOfHistoryPaths.length);+ ByteBuffer buffer = ByteBuffer.wrap(newBytesOfHistoryPaths,+ bytesOfHistoryPaths.length, Integer.BYTES);+ buffer.putInt(oldPathIndex);+ historyPathsStore.put(txn, pathKey,+ new ArrayByteIterable(newBytesOfHistoryPaths));+ }}- } else {- historyPathIndexes = new HashSet<>();}- if (historyPathIndexes.size() < MAX_HISTORY_PATHS) {- int oldPathIndex = getPathIndex(pathToIndexStore, indexToPathStore, txn,- nextIndex, change.getOldPath());- if (!historyPathIndexes.contains(oldPathIndex)) {- historyPathIndexes.add(oldPathIndex);- byte[] newBytesOfHistoryPaths =- new byte[bytesOfHistoryPaths.length+Integer.SIZE];- System.arraycopy(bytesOfHistoryPaths, 0,- newBytesOfHistoryPaths, 0, bytesOfHistoryPaths.length);- ByteBuffer buffer = ByteBuffer.wrap(newBytesOfHistoryPaths,- bytesOfHistoryPaths.length, Integer.BYTES);- buffer.putInt(oldPathIndex);- historyPathsStore.put(txn, pathKey,- new ArrayByteIterable(newBytesOfHistoryPaths));+ }++ if (logCommit.getCommitDate() != null && logCommit.getParentHashes().size() <= 1) {+ int day = new Day(logCommit.getCommitDate()).getValue();++ Map<String, Integer> lineStatsOnDay = lineStats.get(day);+ if (lineStatsOnDay == null) {+ lineStatsOnDay = new HashMap<>();+ lineStats.put(day, lineStatsOnDay);+ }++ Map<String, Integer> languageLines = new HashMap<>();+ for (FileChange change: logCommit.getFileChanges()) {+ int lines = change.getAdditions() - change.getDeletions();+ int lastIndexOfDot = change.getPath().lastIndexOf('.');+ if (lastIndexOfDot != -1 && lines != 0) {+ String fileExt = change.getPath().substring(lastIndexOfDot+1).toLowerCase();+ String language = PROGRAMMING_LANGUAGES.get(fileExt);+ if (language != null) {+ Integer accumulatedLines = languageLines.get(language);+ if (accumulatedLines != null)+ lines += accumulatedLines;+ languageLines.put(language, lines);+ }}}++ for (Map.Entry<String, Integer> entry: languageLines.entrySet()) {+ String language = entry.getKey();+ Integer lines = entry.getValue();+ Integer accumulatedLines = lineStatsOnDay.get(language);+ if (accumulatedLines != null)+ lines += accumulatedLines;+ lineStatsOnDay.put(language, lines);+ }+}++ logCommitOptional = queue.take();}-- }- commitOptional = queue.take();- }- if (logException.get() != null)- throw logException.get();-- writeInt(defaultStore, txn, COMMIT_COUNT_KEY, commitCount);- commitCountCache.remove(project.getId());-- writeInt(defaultStore, txn, NEXT_USER_INDEX_KEY, nextIndex.user);- writeInt(defaultStore, txn, NEXT_PATH_INDEX_KEY, nextIndex.path);-- bytes = SerializationUtils.serialize((Serializable) users);- defaultStore.put(txn, USERS_KEY, new ArrayByteIterable(bytes));- usersCache.remove(project.getId());-- if (files.size() > MAX_COLLECTING_FILES) {- List<String> fileList = new ArrayList<>(files.keySet());- fileList.sort((file1, file2)->files.get(file1).compareTo(files.get(file2)));- for (int i=0; i<fileList.size() - MAX_COLLECTING_FILES; i++)- files.remove(fileList.get(i));- }- bytes = SerializationUtils.serialize((Serializable) files);- defaultStore.put(txn, FILES_KEY, new ArrayByteIterable(bytes));- filesCache.remove(project.getId());-- for (Map.Entry<Integer, Map<Integer, Contribution>> entry: dailyContributionsCache.entrySet()) {- byte[] bytesOfContributionsOnDay = serializeContributions(entry.getValue());- dailyContributionsStore.put(txn, new IntByteIterable(entry.getKey()),- new ArrayByteIterable(bytesOfContributionsOnDay));+ if (logException.get() != null)+ throw logException.get();+ } catch (Exception e) {+ Throwables.propagate(e);+ }++ for (Map.Entry<Integer, Map<Integer, Contribution>> entry: dailyContributionsCache.entrySet()) {+ byte[] bytesOfContributionsOnDay = serializeContributions(entry.getValue());+ dailyContributionsStore.put(txn, new IntByteIterable(entry.getKey()),+ new ArrayByteIterable(bytesOfContributionsOnDay));+ }+ defaultStore.put(txn, OVERALL_CONTRIBUTIONS_KEY,+ new ArrayByteIterable(serializeContributions(overallContributions)));++ bytesOfLineStats = SerializationUtils.serialize((Serializable) lineStats);+ defaultStore.put(txn, LINE_STATS_KEY, new ArrayByteIterable(bytesOfLineStats));++ writeInt(defaultStore, txn, NEXT_USER_INDEX_KEY, nextIndex.user);+ writeInt(defaultStore, txn, NEXT_PATH_INDEX_KEY, nextIndex.path);++ userBytes = SerializationUtils.serialize((Serializable) users);+ defaultStore.put(txn, USERS_KEY, new ArrayByteIterable(userBytes));+ usersCache.remove(project.getId());++ if (files.size() > MAX_COLLECTING_FILES) {+ List<String> fileList = new ArrayList<>(files.keySet());+ fileList.sort((file1, file2)->files.get(file1).compareTo(files.get(file2)));+ for (int i=0; i<fileList.size() - MAX_COLLECTING_FILES; i++)+ files.remove(fileList.get(i));+ }+ fileBytes = SerializationUtils.serialize((Serializable) files);+ defaultStore.put(txn, FILES_KEY, new ArrayByteIterable(fileBytes));+ filesCache.remove(project.getId());++ for (Map.Entry<Long, Integer> entry: editsCache.entrySet())+ writeInt(editsStore, txn, new LongByteIterable(entry.getKey()), entry.getValue());++ byte[] bytes = new byte[20];+ currentCommitId.copyRawTo(bytes, 0);+ defaultStore.put(txn, LAST_COMMIT_OF_DEFAULT_BRANCH_KEY, new ArrayByteIterable(bytes));}- defaultStore.put(txn, OVERALL_CONTRIBUTIONS_KEY,- new ArrayByteIterable(serializeContributions(overallContributions)));-- for (Map.Entry<Long, Integer> entry: editsCache.entrySet())- writeInt(editsStore, txn, new LongByteIterable(entry.getKey()), entry.getValue());-- bytes = new byte[20];- commitId.copyRawTo(bytes, 0);- defaultStore.put(txn, LAST_COMMIT_KEY, new ArrayByteIterable(bytes));- } catch (Exception e) {- Throwables.propagate(e);- }+ });}-- });+ }}}- private void updateContribution(Map<Integer, Contribution> contributions, int key, LogCommit commit) {+ private void updateContribution(Map<Integer, Contribution> contributions, int key, GitCommit commit) {Contribution contribution = contributions.get(key);if (contribution != null) {contribution = new Contribution(@@ -580,6 +834,47 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager implemereturn files;}+ @Override+ public LineStats getLineStats(Project project) {+ Environment env = getEnv(project.getId().toString());+ Store store = getStore(env, DEFAULT_STORE);++ return env.computeInReadonlyTransaction(new TransactionalComputable<LineStats>() {++ @Override+ public LineStats compute(Transaction txn) {+ byte[] bytes = readBytes(store, txn, LINE_STATS_KEY);+ Map<Day, Map<String, Integer>> linesByDay = new HashMap<>();+ if (bytes != null) {+ @SuppressWarnings("unchecked")+ Map<Integer, Map<String, Integer>> storedMap =+ (Map<Integer, Map<String, Integer>>) SerializationUtils.deserialize(bytes);+ List<Integer> sortedDays = new ArrayList<>(storedMap.keySet());+ Collections.sort(sortedDays);+ Map<String, Integer> accumulatedLines = new HashMap<>();+ for (Integer day: sortedDays) {+ Map<String, Integer> linesOnDay = storedMap.get(day);+ for (Map.Entry<String, Integer> entry: linesOnDay.entrySet()) {+ Integer value = accumulatedLines.get(entry.getKey());+ if (value != null)+ value += entry.getValue();+ else+ value = entry.getValue();+ entry.setValue(value);+ }+ for (Map.Entry<String, Integer> entry: accumulatedLines.entrySet())+ linesOnDay.putIfAbsent(entry.getKey(), entry.getValue());+ accumulatedLines = linesOnDay;+ }+ for (Map.Entry<Integer, Map<String, Integer>> entry: storedMap.entrySet())+ linesByDay.put(new Day(entry.getKey()), entry.getValue());+ }+ return new LineStats(linesByDay);+ }++ });+ }+@Overridepublic int getEdits(ProjectFacade project, UserFacade user, String path) {if (user.getEmail() != null) {@@ -618,7 +913,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager implemepublic Set<ObjectId> compute(Transaction txn) {Set<ObjectId> descendants = new HashSet<>();- // use stack instead of recursion to avoid StackOverflowException+ // Use stack instead of recursion to avoid StackOverflowExceptionStack<ObjectId> stack = new Stack<>();descendants.add(ancestor);stack.add(ancestor);@@ -637,10 +932,6 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme}}} else {- /*- * skip the leading byte, which tells whether or not the commit- * has been processed- */for (int i=0; i<(valueBytes.length-1)/20; i++) {ObjectId child = ObjectId.fromRaw(valueBytes, i*20+1);if (!descendants.contains(child)) {@@ -674,14 +965,9 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager implemebyte[] valueBytes = readBytes(store, txn, new ArrayByteIterable(keyBytes));if (valueBytes != null) {if (valueBytes.length % 20 == 0) {- for (int i=0; i<valueBytes.length/20; i++) {+ for (int i=0; i<valueBytes.length/20; i++)children.add(ObjectId.fromRaw(valueBytes, i*20));- }} else {- /*- * skip the leading byte, which tells whether or not the commit- * has been processed- */for (int i=0; i<(valueBytes.length-1)/20; i++) {children.add(ObjectId.fromRaw(valueBytes, i*20+1));}@@ -722,7 +1008,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager implemefor (CollectingWork work: collectingWorks) {logger.debug("Collecting commit information up to ref '{}' in project '{}'...",work.getRefName(), project.getName());- doCollect(project, work.getCommit().copy(), true);+ doCollect(project, work.getCommit().copy(), work.getRefName());}}@@ -733,43 +1019,25 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme}private void collect(Project project) {- Environment env = getEnv(project.getId().toString());- Store commitsStore = getStore(env, COMMITS_STORE);- env.computeInReadonlyTransaction(new TransactionalComputable<Void>() {-- @Override- public Void compute(Transaction txn) {- List<CollectingWork> works = new ArrayList<>();- try (RevWalk revWalk = new RevWalk(project.getRepository())) {- Collection<Ref> refs = new ArrayList<>();- refs.addAll(project.getRepository().getRefDatabase().getRefs(Constants.R_HEADS).values());- refs.addAll(project.getRepository().getRefDatabase().getRefs(Constants.R_TAGS).values());-- for (Ref ref: refs) {- RevObject revObj = revWalk.peel(revWalk.parseAny(ref.getObjectId()));- if (revObj instanceof RevCommit) {- RevCommit commit = (RevCommit) revObj;- byte[] keyBytes = new byte[20];- commit.copyRawTo(keyBytes, 0);- ByteIterable key = new ArrayByteIterable(keyBytes);- byte[] valueBytes = readBytes(commitsStore, txn, key);- if (valueBytes == null || valueBytes.length % 20 == 0)- works.add(new CollectingWork(PRIORITY, commit, ref.getName()));- }- }- } catch (IOException e) {- throw new RuntimeException(e);- }+ List<CollectingWork> works = new ArrayList<>();+ try (RevWalk revWalk = new RevWalk(project.getRepository())) {+ Collection<Ref> refs = new ArrayList<>();+ refs.addAll(project.getRepository().getRefDatabase().getRefs(Constants.R_HEADS).values());+ refs.addAll(project.getRepository().getRefDatabase().getRefs(Constants.R_TAGS).values());- Collections.sort(works, new CommitTimeComparator());-- for (CollectingWork work: works)- batchWorkManager.submit(getBatchWorker(project.getId()), work);-- return null;+ for (Ref ref: refs) {+ RevObject revObj = revWalk.peel(revWalk.parseAny(ref.getObjectId()));+ if (revObj instanceof RevCommit)+ works.add(new CollectingWork(PRIORITY, (RevCommit) revObj, ref.getName()));}-- });+ } catch (IOException e) {+ throw new RuntimeException(e);+ }++ Collections.sort(works, new CommitTimeComparator());++ for (CollectingWork work: works)+ batchWorkManager.submit(getBatchWorker(project.getId()), work);}@Sessionaldiff --git a/core/src/test/java/com/gitplex/server/git/command/LogCommandTest.java b/core/src/test/java/com/gitplex/server/git/command/LogCommandTest.javaindex 1e7a950caa..c8d3b6af62 100644--- a/core/src/test/java/com/gitplex/server/git/command/LogCommandTest.java+++ b/core/src/test/java/com/gitplex/server/git/command/LogCommandTest.java@@ -66,12 +66,12 @@ public class LogCommandTest extends AbstractGitTest {+ "9th line\n");commit("第二个commit");- List<LogCommit> commits = new ArrayList<>();+ List<GitCommit> commits = new ArrayList<>();new LogCommand(git.getRepository().getDirectory()) {@Override- protected void consume(LogCommit commit) {+ protected void consume(GitCommit commit) {commits.add(commit);}@@ -79,7 +79,7 @@ public class LogCommandTest extends AbstractGitTest {assertEquals(2, commits.size());- LogCommit commit = commits.get(0);+ GitCommit commit = commits.get(0);assertEquals(3, commit.getFileChanges().size());diff --git a/web/src/main/java/com/gitplex/server/web/GitPlexUrlMapper.java b/web/src/main/java/com/gitplex/server/web/GitPlexUrlMapper.javaindex 415338a82a..6c595270c2 100644--- a/web/src/main/java/com/gitplex/server/web/GitPlexUrlMapper.java+++ b/web/src/main/java/com/gitplex/server/web/GitPlexUrlMapper.java@@ -44,7 +44,8 @@ import com.gitplex.server.web.page.project.setting.branchprotection.BranchProtecimport com.gitplex.server.web.page.project.setting.commitmessagetransform.CommitMessageTransformPage;import com.gitplex.server.web.page.project.setting.general.GeneralSettingPage;import com.gitplex.server.web.page.project.setting.tagprotection.TagProtectionPage;-import com.gitplex.server.web.page.project.stats.ProjectStatsPage;+import com.gitplex.server.web.page.project.stats.ProjectContribsPage;+import com.gitplex.server.web.page.project.stats.SourceLinesPage;import com.gitplex.server.web.page.project.tags.ProjectTagsPage;import com.gitplex.server.web.page.security.ForgetPage;import com.gitplex.server.web.page.security.LoginPage;@@ -156,7 +157,8 @@ public class GitPlexUrlMapper extends CompoundRequestMapper {add(new GitPlexPageMapper("projects/${project}/commit/${revision}", CommitDetailPage.class));add(new GitPlexPageMapper("projects/${project}/commits", ProjectCommitsPage.class));add(new GitPlexPageMapper("projects/${project}/compare", RevisionComparePage.class));- add(new GitPlexPageMapper("projects/${project}/stats", ProjectStatsPage.class));+ add(new GitPlexPageMapper("projects/${project}/stats/contribs", ProjectContribsPage.class));+ add(new GitPlexPageMapper("projects/${project}/stats/lines", SourceLinesPage.class));add(new GitPlexPageMapper("projects/${project}/branches", ProjectBranchesPage.class));add(new GitPlexPageMapper("projects/${project}/tags", ProjectTagsPage.class));diff --git a/web/src/main/java/com/gitplex/server/web/page/project/ProjectPage.java b/web/src/main/java/com/gitplex/server/web/page/project/ProjectPage.javaindex b555f36a5b..f7fac3f024 100644--- a/web/src/main/java/com/gitplex/server/web/page/project/ProjectPage.java+++ b/web/src/main/java/com/gitplex/server/web/page/project/ProjectPage.java@@ -52,6 +52,7 @@ import com.gitplex.server.web.page.project.pullrequest.requestdetail.RequestDetaimport com.gitplex.server.web.page.project.pullrequest.requestlist.RequestListPage;import com.gitplex.server.web.page.project.setting.ProjectSettingPage;import com.gitplex.server.web.page.project.setting.general.GeneralSettingPage;+import com.gitplex.server.web.page.project.stats.ProjectContribsPage;import com.gitplex.server.web.page.project.stats.ProjectStatsPage;import com.gitplex.server.web.page.project.tags.ProjectTagsPage;import com.google.common.base.Preconditions;@@ -142,7 +143,8 @@ public abstract class ProjectPage extends LayoutPage {tabs.add(new ProjectTab(Model.of("Compare"), "fa fa-fw fa-ext fa-file-diff", 0, RevisionComparePage.class));- tabs.add(new ProjectTab(Model.of("Statistics"), "fa fa-fw fa-bar-chart", 0, ProjectStatsPage.class));+ tabs.add(new ProjectTab(Model.of("Statistics"), "fa fa-fw fa-bar-chart", 0, ProjectContribsPage.class,+ ProjectStatsPage.class));if (SecurityUtils.canManage(getProject()))tabs.add(new ProjectTab(Model.of("Setting"), "fa fa-fw fa-cog", 0, GeneralSettingPage.class, ProjectSettingPage.class));diff --git a/web/src/main/java/com/gitplex/server/web/page/project/stats/ProjectContribsPage.html b/web/src/main/java/com/gitplex/server/web/page/project/stats/ProjectContribsPage.htmlnew file mode 100644index 0000000000..462b79536d--- /dev/null+++ b/web/src/main/java/com/gitplex/server/web/page/project/stats/ProjectContribsPage.html@@ -0,0 +1,38 @@+<wicket:extend>+ <div id="project-contribs">+ <h2>Overall contribution</h2>+ <div class="well">+ <div wicket:id="overallContribution"></div>+ </div>++ <div class="top-contributors">+ <h2>Top 10 User Contributions</h2>+ <form wicket:id="topOptions" class="form-inline">+ <div wicket:id="feedback"></div>+ <div class="form-group">+ <label class="control-label">From Day: </label>+ <input wicket:id="fromDay" placeholder="yyyy-MM-dd" class="form-control"></input>+ </div>+ <div class="form-group">+ <label class="control-label">To Day: </label>+ <input wicket:id="toDay" placeholder="yyyy-MM-dd" class="form-control"></input>+ </div>+ <div class="form-group">+ <label class="control-label">Order By: </label>+ <select wicket:id="orderBy" class="form-control"></select>+ </div>+ <div class="form-group">+ <a wicket:id="update" class="btn btn-default">Update</a>+ </div>+ </form>+ <hr/>+ <div wicket:id="topContributors" class="well">+ <div wicket:id="user"></div>+ <div wicket:id="contributions"></div>+ </div>+ </div>+ </div>+ <wicket:fragment wicket:id="userFrag">+ <a wicket:id="avatar"></a> <a wicket:id="link"></a>+ </wicket:fragment>+</wicket:extend>\ No newline at end of filediff --git a/web/src/main/java/com/gitplex/server/web/page/project/stats/ProjectContribsPage.java b/web/src/main/java/com/gitplex/server/web/page/project/stats/ProjectContribsPage.javanew file mode 100644index 0000000000..a86752a86a--- /dev/null+++ b/web/src/main/java/com/gitplex/server/web/page/project/stats/ProjectContribsPage.java@@ -0,0 +1,173 @@+package com.gitplex.server.web.page.project.stats;++import java.util.ArrayList;+import java.util.Collections;+import java.util.List;+import java.util.Map;++import org.apache.commons.lang.StringUtils;+import org.apache.wicket.markup.head.IHeaderResponse;+import org.apache.wicket.markup.head.OnDomReadyHeaderItem;+import org.apache.wicket.markup.html.form.DropDownChoice;+import org.apache.wicket.markup.html.form.Form;+import org.apache.wicket.markup.html.form.SubmitLink;+import org.apache.wicket.markup.html.form.TextField;+import org.apache.wicket.markup.html.list.ListItem;+import org.apache.wicket.markup.html.list.ListView;+import org.apache.wicket.markup.html.panel.Fragment;+import org.apache.wicket.model.AbstractReadOnlyModel;+import org.apache.wicket.model.IModel;+import org.apache.wicket.model.LoadableDetachableModel;+import org.apache.wicket.model.PropertyModel;+import org.apache.wicket.request.mapper.parameter.PageParameters;++import com.gitplex.server.GitPlex;+import com.gitplex.server.git.Contribution;+import com.gitplex.server.git.Contributor;+import com.gitplex.server.manager.CommitInfoManager;+import com.gitplex.server.util.Day;+import com.gitplex.server.web.WebConstants;+import com.gitplex.server.web.component.avatar.AvatarLink;+import com.gitplex.server.web.component.link.UserLink;++import de.agilecoders.wicket.core.markup.html.bootstrap.common.NotificationPanel;++@SuppressWarnings("serial")+public class ProjectContribsPage extends ProjectStatsPage {++ private final IModel<List<DayContribution>> overallContributionsModel =+ new LoadableDetachableModel<List<DayContribution>>() {++ @Override+ protected List<DayContribution> load() {+ Map<Day, Contribution> map = GitPlex.getInstance(CommitInfoManager.class).getOverallContributions(getProject());+ return sort(map);+ }++ };++ private String fromDayText;++ private String toDayText;++ private String orderBy = Contribution.Type.COMMITS.name();++ public ProjectContribsPage(PageParameters params) {+ super(params);+ }++ @Override+ protected void onDetach() {+ overallContributionsModel.detach();+ super.onDetach();+ }++ private List<DayContribution> sort(Map<Day, Contribution> map) {+ List<DayContribution> list = new ArrayList<>();+ for (Map.Entry<Day, Contribution> entry: map.entrySet())+ list.add(new DayContribution(entry.getKey(), entry.getValue()));+ Collections.sort(list);+ return list;+ }++ @Override+ protected void onInitialize() {+ super.onInitialize();++ add(new ContributionListPanel("overallContribution", overallContributionsModel));++ Form<?> form = new Form<Void>("topOptions");+ form.add(new SubmitLink("update") {++ @Override+ public void onSubmit() {+ super.onSubmit();++ if (StringUtils.isNotBlank(fromDayText)) {+ try {+ WebConstants.DATE_FORMATTER.parseDateTime(fromDayText);+ } catch (IllegalArgumentException e) {+ error("Invalid fromDay");+ }+ }++ if (StringUtils.isNotBlank(toDayText)) {+ try {+ WebConstants.DATE_FORMATTER.parseDateTime(toDayText);+ } catch (IllegalArgumentException e) {+ error("Invalid toDay");+ }+ }+ }++ });+ form.add(new NotificationPanel("feedback", form));++ form.add(new TextField<String>("fromDay", new PropertyModel<String>(this, "fromDayText")));+ form.add(new TextField<String>("toDay", new PropertyModel<String>(this, "toDayText")));+ List<String> orderByChoices = new ArrayList<>();+ for (Contribution.Type type: Contribution.Type.values())+ orderByChoices.add(type.name());++ form.add(new DropDownChoice<String>("orderBy", new PropertyModel<String>(this, "orderBy"), orderByChoices));++ add(form);++ add(new ListView<Contributor>("topContributors", new LoadableDetachableModel<List<Contributor>>() {++ @Override+ protected List<Contributor> load() {+ List<DayContribution> overallContributions = overallContributionsModel.getObject();+ int size = overallContributions.size();+ if (size != 0) {+ Day fromDay;+ if (StringUtils.isNotBlank(fromDayText)) {+ fromDay = new Day(WebConstants.DATE_FORMATTER.parseDateTime(fromDayText));+ } else {+ fromDay = overallContributions.get(0).getDay();+ }++ Day toDay;+ if (StringUtils.isNotBlank(toDayText)) {+ toDay = new Day(WebConstants.DATE_FORMATTER.parseDateTime(toDayText));+ } else {+ toDay = overallContributions.get(size-1).getDay();+ }++ return GitPlex.getInstance(CommitInfoManager.class)+ .getTopContributors(getProject(), 100, Contribution.Type.valueOf(orderBy), fromDay, toDay);+ } else {+ return new ArrayList<>();+ }+ }++ }) {++ @Override+ protected void populateItem(ListItem<Contributor> item) {+ Contributor userContribution = item.getModelObject();+ Fragment fragment = new Fragment("user", "userFrag", ProjectContribsPage.this);+ fragment.add(new AvatarLink("avatar", userContribution.getUser()));+ fragment.add(new UserLink("link", userContribution.getUser()));+ item.add(fragment);++ item.add(new ContributionListPanel("contributions", new AbstractReadOnlyModel<List<DayContribution>>() {++ @Override+ public List<DayContribution> getObject() {+ return sort(item.getModelObject().getDailyContributions());+ }++ }));+ }++ });+ }++ @Override+ public void renderHead(IHeaderResponse response) {+ super.renderHead(response);+ response.render(OnDomReadyHeaderItem.forScript("gitplex.server.stats.contribs.onDomReady();"));+ }++}diff --git a/web/src/main/java/com/gitplex/server/web/page/project/stats/ProjectStatsPage.html b/web/src/main/java/com/gitplex/server/web/page/project/stats/ProjectStatsPage.htmlindex 91ccbea88c..efd2a2185f 100644--- a/web/src/main/java/com/gitplex/server/web/page/project/stats/ProjectStatsPage.html+++ b/web/src/main/java/com/gitplex/server/web/page/project/stats/ProjectStatsPage.html@@ -1,38 +1,6 @@<wicket:extend><div id="project-stats">- <div class="well">- <h2>Overall contribution</h2>- <div wicket:id="overallContribution"></div>- </div>-- <div class="top-contributors">- <h2>Top 10 User Contributions</h2>- <form wicket:id="topOptions" class="form-inline">- <div wicket:id="feedback"></div>- <div class="form-group">- <label class="control-label">From Day: </label>- <input wicket:id="fromDay" placeholder="yyyy-MM-dd" class="form-control"></input>- </div>- <div class="form-group">- <label class="control-label">To Day: </label>- <input wicket:id="toDay" placeholder="yyyy-MM-dd" class="form-control"></input>- </div>- <div class="form-group">- <label class="control-label">Order By: </label>- <select wicket:id="orderBy" class="form-control"></select>- </div>- <div class="form-group">- <a wicket:id="update" class="btn btn-default">Update</a>- </div>- </form>- <hr/>- <div wicket:id="topContributors" class="well">- <div wicket:id="user"></div>- <div wicket:id="contributions"></div>- </div>- </div>+ <ul wicket:id="statsTabs" class="tabs nav nav-tabs"></ul>+ <wicket:child></wicket:child></div>- <wicket:fragment wicket:id="userFrag">- <a wicket:id="avatar"></a> <a wicket:id="link"></a>- </wicket:fragment></wicket:extend>\ No newline at end of filediff --git a/web/src/main/java/com/gitplex/server/web/page/project/stats/ProjectStatsPage.java b/web/src/main/java/com/gitplex/server/web/page/project/stats/ProjectStatsPage.javaindex ccadf069d7..c406659f68 100644--- a/web/src/main/java/com/gitplex/server/web/page/project/stats/ProjectStatsPage.java+++ b/web/src/main/java/com/gitplex/server/web/page/project/stats/ProjectStatsPage.java@@ -1,176 +1,76 @@package com.gitplex.server.web.page.project.stats;import java.util.ArrayList;-import java.util.Collections;import java.util.List;-import java.util.Map;-import org.apache.commons.lang.StringUtils;+import org.apache.wicket.Component;+import org.apache.wicket.Page;import org.apache.wicket.markup.head.IHeaderResponse;import org.apache.wicket.markup.head.JavaScriptHeaderItem;-import org.apache.wicket.markup.head.OnDomReadyHeaderItem;-import org.apache.wicket.markup.html.form.DropDownChoice;-import org.apache.wicket.markup.html.form.Form;-import org.apache.wicket.markup.html.form.SubmitLink;-import org.apache.wicket.markup.html.form.TextField;-import org.apache.wicket.markup.html.list.ListItem;-import org.apache.wicket.markup.html.list.ListView;-import org.apache.wicket.markup.html.panel.Fragment;-import org.apache.wicket.model.AbstractReadOnlyModel;-import org.apache.wicket.model.IModel;-import org.apache.wicket.model.LoadableDetachableModel;-import org.apache.wicket.model.PropertyModel;+import org.apache.wicket.markup.html.link.Link;+import org.apache.wicket.model.Model;import org.apache.wicket.request.mapper.parameter.PageParameters;-import com.gitplex.server.GitPlex;-import com.gitplex.server.git.Contribution;-import com.gitplex.server.git.Contributor;-import com.gitplex.server.manager.CommitInfoManager;-import com.gitplex.server.util.Day;-import com.gitplex.server.web.WebConstants;-import com.gitplex.server.web.component.avatar.AvatarLink;-import com.gitplex.server.web.component.link.UserLink;+import com.gitplex.server.web.component.link.ViewStateAwarePageLink;+import com.gitplex.server.web.component.tabbable.PageTab;+import com.gitplex.server.web.component.tabbable.PageTabLink;+import com.gitplex.server.web.component.tabbable.Tab;+import com.gitplex.server.web.component.tabbable.Tabbable;import com.gitplex.server.web.page.project.ProjectPage;-import de.agilecoders.wicket.core.markup.html.bootstrap.common.NotificationPanel;-@SuppressWarnings("serial")-public class ProjectStatsPage extends ProjectPage {-- private final IModel<List<DayContribution>> overallContributionsModel =- new LoadableDetachableModel<List<DayContribution>>() {-- @Override- protected List<DayContribution> load() {- Map<Day, Contribution> map = GitPlex.getInstance(CommitInfoManager.class).getOverallContributions(getProject());- return sort(map);- }-- };+public abstract class ProjectStatsPage extends ProjectPage {- private String fromDayText;-- private String toDayText;-- private String orderBy = Contribution.Type.COMMITS.name();-public ProjectStatsPage(PageParameters params) {super(params);}- @Override- protected void onDetach() {- overallContributionsModel.detach();- super.onDetach();- }-- private List<DayContribution> sort(Map<Day, Contribution> map) {- List<DayContribution> list = new ArrayList<>();- for (Map.Entry<Day, Contribution> entry: map.entrySet())- list.add(new DayContribution(entry.getKey(), entry.getValue()));- Collections.sort(list);- return list;- }-@Overrideprotected void onInitialize() {super.onInitialize();- add(new ContributionListPanel("overallContribution", overallContributionsModel));+ List<Tab> tabs = new ArrayList<>();- Form<?> form = new Form<Void>("topOptions");- form.add(new SubmitLink("update") {+ tabs.add(new PageTab(Model.of("Contributions"), ProjectContribsPage.class) {@Override- public void onSubmit() {- super.onSubmit();-- if (StringUtils.isNotBlank(fromDayText)) {- try {- WebConstants.DATE_FORMATTER.parseDateTime(fromDayText);- } catch (IllegalArgumentException e) {- error("Invalid fromDay");- }- }-- if (StringUtils.isNotBlank(toDayText)) {- try {- WebConstants.DATE_FORMATTER.parseDateTime(toDayText);- } catch (IllegalArgumentException e) {- error("Invalid toDay");- }- }- }-- });- form.add(new NotificationPanel("feedback", form));-- form.add(new TextField<String>("fromDay", new PropertyModel<String>(this, "fromDayText")));- form.add(new TextField<String>("toDay", new PropertyModel<String>(this, "toDayText")));- List<String> orderByChoices = new ArrayList<>();- for (Contribution.Type type: Contribution.Type.values())- orderByChoices.add(type.name());-- form.add(new DropDownChoice<String>("orderBy", new PropertyModel<String>(this, "orderBy"), orderByChoices));-- add(form);-- add(new ListView<Contributor>("topContributors", new LoadableDetachableModel<List<Contributor>>() {+ public Component render(String componentId) {+ return new PageTabLink(componentId, this) {- @Override- protected List<Contributor> load() {- List<DayContribution> overallContributions = overallContributionsModel.getObject();- int size = overallContributions.size();- if (size != 0) {- Day fromDay;- if (StringUtils.isNotBlank(fromDayText)) {- fromDay = new Day(WebConstants.DATE_FORMATTER.parseDateTime(fromDayText));- } else {- fromDay = overallContributions.get(0).getDay();- }-- Day toDay;- if (StringUtils.isNotBlank(toDayText)) {- toDay = new Day(WebConstants.DATE_FORMATTER.parseDateTime(toDayText));- } else {- toDay = overallContributions.get(size-1).getDay();+ @Override+ protected Link<?> newLink(String linkId, Class<? extends Page> pageClass) {+ return new ViewStateAwarePageLink<>(linkId, ProjectContribsPage.class,+ ProjectContribsPage.paramsOf(getProject()));}- return GitPlex.getInstance(CommitInfoManager.class)- .getTopContributors(getProject(), 25, Contribution.Type.valueOf(orderBy), fromDay, toDay);- } else {- return new ArrayList<>();- }+ };}- }) {-+ });+ tabs.add(new PageTab(Model.of("Source Lines"), SourceLinesPage.class) {+@Override- protected void populateItem(ListItem<Contributor> item) {- Contributor userContribution = item.getModelObject();- Fragment fragment = new Fragment("user", "userFrag", ProjectStatsPage.this);- fragment.add(new AvatarLink("avatar", userContribution.getUser()));- fragment.add(new UserLink("link", userContribution.getUser()));- item.add(fragment);-- item.add(new ContributionListPanel("contributions", new AbstractReadOnlyModel<List<DayContribution>>() {+ public Component render(String componentId) {+ return new PageTabLink(componentId, this) {@Override- public List<DayContribution> getObject() {- return sort(item.getModelObject().getDailyContributions());+ protected Link<?> newLink(String linkId, Class<? extends Page> pageClass) {+ return new ViewStateAwarePageLink<>(linkId, SourceLinesPage.class,+ SourceLinesPage.paramsOf(getProject()));}- }));+ };}});++ add(new Tabbable("statsTabs", tabs).setOutputMarkupId(true));}@Overridepublic void renderHead(IHeaderResponse response) {super.renderHead(response);response.render(JavaScriptHeaderItem.forReference(new ProjectStatsResourceReference()));- response.render(OnDomReadyHeaderItem.forScript("gitplex.server.stats.onDomReady();"));}}diff --git a/web/src/main/java/com/gitplex/server/web/page/project/stats/SourceLinesPage.html b/web/src/main/java/com/gitplex/server/web/page/project/stats/SourceLinesPage.htmlnew file mode 100644index 0000000000..b508c60ab7--- /dev/null+++ b/web/src/main/java/com/gitplex/server/web/page/project/stats/SourceLinesPage.html@@ -0,0 +1,8 @@+<wicket:extend>+ <div id="source-lines">+ <div wicket:id="languages" class="well">+ <h4 wicket:id="language"></h4>+ <table wicket:id="lines"></table>+ </div>+ </div>+</wicket:extend>\ No newline at end of filediff --git a/web/src/main/java/com/gitplex/server/web/page/project/stats/SourceLinesPage.java b/web/src/main/java/com/gitplex/server/web/page/project/stats/SourceLinesPage.javanew file mode 100644index 0000000000..6ea132a716--- /dev/null+++ b/web/src/main/java/com/gitplex/server/web/page/project/stats/SourceLinesPage.java@@ -0,0 +1,164 @@+package com.gitplex.server.web.page.project.stats;++import java.util.ArrayList;+import java.util.Collections;+import java.util.Comparator;+import java.util.Iterator;+import java.util.List;+import java.util.Map;++import org.apache.wicket.extensions.ajax.markup.html.repeater.data.table.AjaxNavigationToolbar;+import org.apache.wicket.extensions.markup.html.repeater.data.grid.ICellPopulator;+import org.apache.wicket.extensions.markup.html.repeater.data.table.AbstractColumn;+import org.apache.wicket.extensions.markup.html.repeater.data.table.DataTable;+import org.apache.wicket.extensions.markup.html.repeater.data.table.HeadersToolbar;+import org.apache.wicket.extensions.markup.html.repeater.data.table.IColumn;+import org.apache.wicket.extensions.markup.html.repeater.util.SortableDataProvider;+import org.apache.wicket.markup.head.IHeaderResponse;+import org.apache.wicket.markup.head.OnDomReadyHeaderItem;+import org.apache.wicket.markup.html.basic.Label;+import org.apache.wicket.markup.html.list.ListItem;+import org.apache.wicket.markup.html.list.ListView;+import org.apache.wicket.markup.repeater.Item;+import org.apache.wicket.model.IModel;+import org.apache.wicket.model.LoadableDetachableModel;+import org.apache.wicket.model.Model;+import org.apache.wicket.request.mapper.parameter.PageParameters;++import com.gitplex.server.GitPlex;+import com.gitplex.server.git.LineStats;+import com.gitplex.server.manager.CommitInfoManager;+import com.gitplex.server.util.Day;+import com.gitplex.server.web.WebConstants;+import com.gitplex.server.web.util.DateUtils;++@SuppressWarnings("serial")+public class SourceLinesPage extends ProjectStatsPage {++ private IModel<LineStats> lineStatsModel = new LoadableDetachableModel<LineStats>() {++ @Override+ protected LineStats load() {+ return GitPlex.getInstance(CommitInfoManager.class).getLineStats(getProject());+ }++ };++ public SourceLinesPage(PageParameters params) {+ super(params);+ }++ @Override+ protected void onDetach() {+ lineStatsModel.detach();+ super.onDetach();+ }++ private LineStats getLineStats() {+ return lineStatsModel.getObject();+ }++ @Override+ protected void onInitialize() {+ super.onInitialize();++ add(new ListView<String>("languages", new LoadableDetachableModel<List<String>>() {++ @Override+ protected List<String> load() {+ Day lastDay = getLineStats().getLastDay();+ if (lastDay != null) {+ Map<String, Integer> linesOnDay = getLineStats().getLinesByDay().get(lastDay);+ List<String> languages = new ArrayList<>(linesOnDay.keySet());+ Collections.sort(languages, new Comparator<String>() {++ @Override+ public int compare(String o1, String o2) {+ return linesOnDay.get(o2) - linesOnDay.get(o1);+ }++ });+ return languages;+ } else {+ return new ArrayList<>();+ }+ }++ }) {++ @Override+ protected void populateItem(ListItem<String> item) {+ item.add(new Label("language", item.getModelObject()));++ List<IColumn<Day, Void>> columns = new ArrayList<>();++ columns.add(new AbstractColumn<Day, Void>(Model.of("Date")) {++ @Override+ public void populateItem(Item<ICellPopulator<Day>> cellItem, String componentId,+ IModel<Day> rowModel) {+ cellItem.add(new Label(componentId, DateUtils.formatDate(rowModel.getObject().getDate())));+ }+ });++ columns.add(new AbstractColumn<Day, Void>(Model.of("Lines")) {++ @Override+ public void populateItem(Item<ICellPopulator<Day>> cellItem, String componentId,+ IModel<Day> rowModel) {+ String language = item.getModelObject();+ Day day = rowModel.getObject();+ int lines = getLineStats().getLinesByLanguage().get(language).get(day);+ cellItem.add(new Label(componentId, lines));+ }++ });++ SortableDataProvider<Day, Void> dataProvider = new SortableDataProvider<Day, Void>() {++ private List<Day> getDays() {+ List<Day> days = new ArrayList<>();+ days = new ArrayList<>(getLineStats().getLinesByLanguage().get(item.getModelObject()).keySet());+ Collections.sort(days);+ return days;+ }++ @Override+ public Iterator<? extends Day> iterator(long first, long count) {+ List<Day> days = getDays();+ if (first + count <= days.size())+ return days.subList((int)first, (int)(first+count)).iterator();+ else+ return days.subList((int)first, days.size()).iterator();+ }++ @Override+ public long size() {+ return getDays().size();+ }++ @Override+ public IModel<Day> model(Day object) {+ return Model.of(object);+ }+ };++ DataTable<Day, Void> linesTable =+ new DataTable<Day, Void>("lines", columns, dataProvider, WebConstants.PAGE_SIZE);++ linesTable.addTopToolbar(new HeadersToolbar<>(linesTable, dataProvider));+ linesTable.addBottomToolbar(new AjaxNavigationToolbar(linesTable));+ linesTable.setOutputMarkupId(true);+ item.add(linesTable);+ }++ });+ }++ @Override+ public void renderHead(IHeaderResponse response) {+ super.renderHead(response);+ response.render(OnDomReadyHeaderItem.forScript("gitplex.server.stats.lineStats.onDomReady();"));+ }++}diff --git a/web/src/main/java/com/gitplex/server/web/page/project/stats/project-stats.css b/web/src/main/java/com/gitplex/server/web/page/project/stats/project-stats.cssindex ee9f3ffbd8..5e18f5dda2 100644--- a/web/src/main/java/com/gitplex/server/web/page/project/stats/project-stats.css+++ b/web/src/main/java/com/gitplex/server/web/page/project/stats/project-stats.css@@ -2,10 +2,10 @@padding: 20px;}-#project-stats .form-inline div {+#project-contribs .form-inline div {margin-right: 10px;}-#project-stats .form-inline div .btn {+#project-contribs .form-inline div .btn {margin-left: 20px;}\ No newline at end of filediff --git a/web/src/main/java/com/gitplex/server/web/page/project/stats/project-stats.js b/web/src/main/java/com/gitplex/server/web/page/project/stats/project-stats.jsindex 2b13258cd6..7e9bb89f0f 100644--- a/web/src/main/java/com/gitplex/server/web/page/project/stats/project-stats.js+++ b/web/src/main/java/com/gitplex/server/web/page/project/stats/project-stats.js@@ -1,5 +1,12 @@gitplex.server.stats = {- onDomReady: function() {-+ contribs: {+ onDomReady: function() {++ }+ },+ sourceLines: {+ onDomReady: function() {++ }}}