The Unified and Autonomous Development Platform
Fix issue #786 - Path suggestion in commit query should not contains old paths in history
Robin Shenfe8386841cdf
diff --git a/server-core/src/main/java/io/onedev/server/git/command/ListChangedFilesCommand.java b/server-core/src/main/java/io/onedev/server/git/command/ListChangedFilesCommand.javaindex 7663908a0d..c064028dde 100644--- a/server-core/src/main/java/io/onedev/server/git/command/ListChangedFilesCommand.java+++ b/server-core/src/main/java/io/onedev/server/git/command/ListChangedFilesCommand.java@@ -23,8 +23,6 @@ public class ListChangedFilesCommand extends GitCommand<Collection<String>> {private String toRev;- private String path;-public ListChangedFilesCommand(File gitDir, Map<String, String> gitEnvs) {super(gitDir, gitEnvs);}@@ -39,11 +37,6 @@ public class ListChangedFilesCommand extends GitCommand<Collection<String>> {return this;}- public ListChangedFilesCommand path(String path) {- this.path = path;- return this;- }-@Overridepublic Collection<String> call() {Preconditions.checkNotNull(toRev, "toRev has to be specified.");@@ -55,9 +48,6 @@ public class ListChangedFilesCommand extends GitCommand<Collection<String>> {cmd.addArgs("diff", "--name-only", "--no-renames", fromRev + ".." + toRev);- if (path != null)- cmd.addArgs("--", path);-cmd.execute(new LineConsumer() {@Overridediff --git a/server-core/src/main/java/io/onedev/server/git/command/ListFileChangesCommand.java b/server-core/src/main/java/io/onedev/server/git/command/ListFileChangesCommand.javanew file mode 100644index 0000000000..e5c25d3e78--- /dev/null+++ b/server-core/src/main/java/io/onedev/server/git/command/ListFileChangesCommand.java@@ -0,0 +1,81 @@+package io.onedev.server.git.command;++import java.io.File;+import java.util.ArrayList;+import java.util.Collection;+import java.util.List;+import java.util.StringTokenizer;++import org.eclipse.jgit.util.QuotedString;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++import com.google.common.base.Preconditions;++import io.onedev.commons.utils.command.Commandline;+import io.onedev.commons.utils.command.LineConsumer;++public class ListFileChangesCommand extends GitCommand<Collection<FileChange>> {++ private static final Logger logger = LoggerFactory.getLogger(ListFileChangesCommand.class);++ private String fromRev;++ private String toRev;++ public ListFileChangesCommand(File gitDir) {+ super(gitDir);+ }++ public ListFileChangesCommand fromRev(String fromRev) {+ this.fromRev = fromRev;+ return this;+ }++ public ListFileChangesCommand toRev(String toRev) {+ this.toRev = toRev;+ return this;+ }++ @Override+ public Collection<FileChange> call() {+ Preconditions.checkNotNull(toRev, "toRev has to be specified.");+ Preconditions.checkNotNull(fromRev, "fromRev has to be specified.");++ List<FileChange> fileChanges = new ArrayList<>();++ Commandline cmd = cmd();++ cmd.addArgs("diff", "--name-status", "--no-renames", fromRev + ".." + toRev);++ cmd.execute(new LineConsumer() {++ @Override+ public void consume(String line) {+ StringTokenizer tokenizer = new StringTokenizer(line, "\t");+ String statusCode = tokenizer.nextToken();+ if (statusCode.equals("M") || statusCode.equals("T")) {+ String path = QuotedString.GIT_PATH.dequote(tokenizer.nextToken("\t"));+ fileChanges.add(new FileChange(path, path, -1, -1));+ } else if (statusCode.equals("D")) {+ String oldPath = QuotedString.GIT_PATH.dequote(tokenizer.nextToken("\t"));+ fileChanges.add(new FileChange(oldPath, null, -1, -1));+ } else if (statusCode.equals("A")) {+ String newPath = QuotedString.GIT_PATH.dequote(tokenizer.nextToken("\t"));+ fileChanges.add(new FileChange(null, newPath, -1, -1));+ }+ }++ }, new LineConsumer() {++ @Override+ public void consume(String line) {+ logger.error(line);+ }++ }).checkReturnCode();++ return fileChanges;+ }++}diff --git a/server-core/src/main/java/io/onedev/server/git/command/ListFilesCommand.java b/server-core/src/main/java/io/onedev/server/git/command/ListFilesCommand.javanew file mode 100644index 0000000000..f57f3786c5--- /dev/null+++ b/server-core/src/main/java/io/onedev/server/git/command/ListFilesCommand.java@@ -0,0 +1,62 @@+package io.onedev.server.git.command;++import java.io.File;+import java.util.Collection;+import java.util.HashSet;+import java.util.Set;++import org.eclipse.jgit.util.QuotedString;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++import com.google.common.base.Preconditions;++import io.onedev.commons.utils.command.Commandline;+import io.onedev.commons.utils.command.LineConsumer;++public class ListFilesCommand extends GitCommand<Collection<String>> {++ private static final Logger logger = LoggerFactory.getLogger(ListFilesCommand.class);++ private String revision;++ public ListFilesCommand(File gitDir) {+ super(gitDir);+ }++ public ListFilesCommand revision(String revision) {+ this.revision = revision;+ return this;+ }++ @Override+ public Collection<String> call() {+ Preconditions.checkNotNull(revision, "revision has to be specified.");++ Set<String> files = new HashSet<String>();++ Commandline cmd = cmd();++ cmd.addArgs("ls-tree", "--name-only", "-r", revision);++ cmd.execute(new LineConsumer() {++ @Override+ public void consume(String line) {+ if (line.trim().length() != 0)+ files.add(QuotedString.GIT_PATH.dequote(line));+ }++ }, new LineConsumer() {++ @Override+ public void consume(String line) {+ logger.error(line);+ }++ }).checkReturnCode();++ return files;+ }++}diff --git a/server-core/src/main/java/io/onedev/server/infomanager/DefaultCommitInfoManager.java b/server-core/src/main/java/io/onedev/server/infomanager/DefaultCommitInfoManager.javaindex 95b5f2a130..20b92008d1 100644--- a/server-core/src/main/java/io/onedev/server/infomanager/DefaultCommitInfoManager.java+++ b/server-core/src/main/java/io/onedev/server/infomanager/DefaultCommitInfoManager.java@@ -55,6 +55,8 @@ import io.onedev.server.git.GitContributor;import io.onedev.server.git.GitUtils;import io.onedev.server.git.command.FileChange;import io.onedev.server.git.command.GitCommit;+import io.onedev.server.git.command.ListFileChangesCommand;+import io.onedev.server.git.command.ListFilesCommand;import io.onedev.server.git.command.ListNumStatsCommand;import io.onedev.server.git.command.LogCommand;import io.onedev.server.git.command.RevListCommand;@@ -89,14 +91,12 @@ public class DefaultCommitInfoManager extends AbstractMultiEnvironmentManager imprivate static final Logger logger = LoggerFactory.getLogger(DefaultCommitInfoManager.class);- private static final int INFO_VERSION = 13;+ private static final int INFO_VERSION = 14;private static final long LOG_FILE_SIZE = 256*1024;private static final int COLLECT_BATCH_SIZE = 10000;- private static final int MAX_COLLECTING_FILES = 50000;-private static final int MAX_HISTORY_PATHS = 100;private static final int MAX_COMMIT_FILES = 100;@@ -137,6 +137,8 @@ public class DefaultCommitInfoManager extends AbstractMultiEnvironmentManager imprivate static final ByteIterable LAST_COMMIT_OF_CONTRIBS_KEY = new StringByteIterable("lastCommitOfContribs");+ private static final ByteIterable LAST_COMMIT_OF_FILES_KEY = new StringByteIterable("lastCommitOfFiles");+private static final ByteIterable LINE_STATS_KEY = new StringByteIterable("lineStats");private static final ByteIterable USERS_KEY = new StringByteIterable("users");@@ -252,13 +254,6 @@ public class DefaultCommitInfoManager extends AbstractMultiEnvironmentManager imelseusers = new HashSet<>();- Map<String, Long> files;- byte[] fileBytes = readBytes(defaultStore, txn, FILES_KEY);- if (fileBytes != null)- files = (Map<String, Long>) SerializationUtils.deserialize(fileBytes);- else- files = new HashMap<>();-new ElementPumper<GitCommit>() {@Override@@ -337,11 +332,6 @@ public class DefaultCommitInfoManager extends AbstractMultiEnvironmentManager imwriteCommits(fixCommitsStore, txn, issueKey, fixingCommits);}- if (currentCommit.getCommitDate() != null) {- for (String file: currentCommit.getChangedFiles())- files.put(file, currentCommit.getCommitDate().getTime());- }-if (currentCommit.getCommitter() != null)users.add(new NameAndEmail(currentCommit.getCommitter()));@@ -434,16 +424,6 @@ public class DefaultCommitInfoManager extends AbstractMultiEnvironmentManager imdefaultStore.put(txn, USERS_KEY, new ArrayByteIterable(userBytes));usersCache.remove(project.getId());- if (files.size() > MAX_COLLECTING_FILES) {- List<String> fileList = new ArrayList<>(files.keySet());- fileList.sort((file1, file2)->files.get(file1).compareTo(files.get(file2)));- for (int i=0; i<fileList.size() - MAX_COLLECTING_FILES; i++)- files.remove(fileList.get(i));- }- fileBytes = SerializationUtils.serialize((Serializable) files);- defaultStore.put(txn, FILES_KEY, new ArrayByteIterable(fileBytes));- filesCache.remove(project.getId());-for (Map.Entry<Long, Integer> entry: commitCountCache.entrySet())writeInt(commitCountsStore, txn, new LongByteIterable(entry.getKey()), entry.getValue());@@ -458,6 +438,7 @@ public class DefaultCommitInfoManager extends AbstractMultiEnvironmentManager imif (GitUtils.branch2ref(project.getDefaultBranch()).equals(refName)) {collectLineStats(project, commitId);collectContribs(project, commitId);+ collectFiles(project, commitId);}logger.debug("Collected commit information (project: {}, ref: {})", project.getPath(), refName);@@ -573,6 +554,85 @@ public class DefaultCommitInfoManager extends AbstractMultiEnvironmentManager im});}+ private void collectFiles(Project project, ObjectId commitId) {+ Environment env = getEnv(project.getId().toString());+ Store defaultStore = getStore(env, DEFAULT_STORE);++ Repository repository = project.getRepository();++ ObjectId lastCommitId = env.computeInTransaction(new TransactionalComputable<ObjectId>() {++ @Override+ public ObjectId compute(Transaction txn) {+ ObjectId lastCommitId;+ byte[] lastCommitBytes = readBytes(defaultStore, txn, LAST_COMMIT_OF_FILES_KEY);+ if (lastCommitBytes != null) {+ lastCommitId = ObjectId.fromRaw(lastCommitBytes);+ try {+ if (!repository.getObjectDatabase().has(lastCommitId))+ lastCommitId = null;+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ } else {+ lastCommitId = null;+ }+ return lastCommitId;+ }+ });++ if (lastCommitId == null) {+ env.executeInTransaction(new TransactionalExecutable() {++ @Override+ public void execute(Transaction txn) {+ ListFilesCommand command = new ListFilesCommand(project.getGitDir());+ Collection<String> files = command.revision(commitId.name()).call();++ byte[] bytesOfFiles = SerializationUtils.serialize((Serializable) files);+ defaultStore.put(txn, FILES_KEY, new ArrayByteIterable(bytesOfFiles));+ defaultStore.put(txn, LAST_COMMIT_OF_FILES_KEY, new CommitByteIterable(commitId));+ filesCache.remove(project.getId());+ }++ });+ } else {+ env.executeInTransaction(new TransactionalExecutable() {++ @SuppressWarnings("unchecked")+ @Override+ public void execute(Transaction txn) {+ Collection<String> files;+ byte[] bytesOfFiles = readBytes(defaultStore, txn, FILES_KEY);+ if (bytesOfFiles != null)+ files = (Set<String>) SerializationUtils.deserialize(bytesOfFiles);+ else+ files = new HashSet<>();++ boolean filesChanged = false;+ ListFileChangesCommand command = new ListFileChangesCommand(project.getGitDir());+ for (FileChange change: command.fromRev(lastCommitId.name()).toRev(commitId.name()).call()) {+ if (change.getOldPath() == null && change.getNewPath() != null) {+ files.add(change.getNewPath());+ filesChanged = true;+ } else if (change.getOldPath() != null && change.getNewPath() == null) {+ files.remove(change.getOldPath());+ filesChanged = true;+ }+ }++ if (filesChanged) {+ bytesOfFiles = SerializationUtils.serialize((Serializable) files);+ defaultStore.put(txn, FILES_KEY, new ArrayByteIterable(bytesOfFiles));+ defaultStore.put(txn, LAST_COMMIT_OF_FILES_KEY, new CommitByteIterable(commitId));+ filesCache.remove(project.getId());+ }+ }++ });+ }+ }+private void collectLineStats(Project project, ObjectId commitId) {Environment env = getEnv(project.getId().toString());Store defaultStore = getStore(env, DEFAULT_STORE);@@ -672,7 +732,6 @@ public class DefaultCommitInfoManager extends AbstractMultiEnvironmentManager im}});-}}@@ -818,7 +877,7 @@ public class DefaultCommitInfoManager extends AbstractMultiEnvironmentManager imList<String> files = filesCache.get(project.getId());if (files == null) {Environment env = getEnv(project.getId().toString());- final Store store = getStore(env, DEFAULT_STORE);+ Store store = getStore(env, DEFAULT_STORE);files = env.computeInReadonlyTransaction(new TransactionalComputable<List<String>>() {@@ -827,13 +886,11 @@ public class DefaultCommitInfoManager extends AbstractMultiEnvironmentManager impublic List<String> compute(Transaction txn) {byte[] bytes = readBytes(store, txn, FILES_KEY);if (bytes != null) {- List<String> files = new ArrayList<>(- ((Map<String, Long>)SerializationUtils.deserialize(bytes)).keySet());+ List<String> files = new ArrayList<>((Collection<String>)SerializationUtils.deserialize(bytes));Map<String, List<String>> segmentsMap = new HashMap<>();Splitter splitter = Splitter.on("/");- for (String file: files) {+ for (String file: files)segmentsMap.put(file, splitter.splitToList(file));- }files.sort(new Comparator<String>() {@Override