Skip to content

The Unified and Autonomous Development Platform

Fix issue #786 - Path suggestion in commit query should not contains old paths in history

Robin Shenfe8386841cdf
diff --git a/server-core/src/main/java/io/onedev/server/git/command/ListChangedFilesCommand.java b/server-core/src/main/java/io/onedev/server/git/command/ListChangedFilesCommand.java
index 7663908a0d..c064028dde 100644
--- a/server-core/src/main/java/io/onedev/server/git/command/ListChangedFilesCommand.java
+++ b/server-core/src/main/java/io/onedev/server/git/command/ListChangedFilesCommand.java
@@ -23,8 +23,6 @@ public class ListChangedFilesCommand extends GitCommand<Collection<String>> {
private String toRev;
- private String path;
-
public ListChangedFilesCommand(File gitDir, Map<String, String> gitEnvs) {
super(gitDir, gitEnvs);
}
@@ -39,11 +37,6 @@ public class ListChangedFilesCommand extends GitCommand<Collection<String>> {
return this;
}
- public ListChangedFilesCommand path(String path) {
- this.path = path;
- return this;
- }
-
@Override
public Collection<String> call() {
Preconditions.checkNotNull(toRev, "toRev has to be specified.");
@@ -55,9 +48,6 @@ public class ListChangedFilesCommand extends GitCommand<Collection<String>> {
cmd.addArgs("diff", "--name-only", "--no-renames", fromRev + ".." + toRev);
- if (path != null)
- cmd.addArgs("--", path);
-
cmd.execute(new LineConsumer() {
@Override
diff --git a/server-core/src/main/java/io/onedev/server/git/command/ListFileChangesCommand.java b/server-core/src/main/java/io/onedev/server/git/command/ListFileChangesCommand.java
new file mode 100644
index 0000000000..e5c25d3e78
--- /dev/null
+++ b/server-core/src/main/java/io/onedev/server/git/command/ListFileChangesCommand.java
@@ -0,0 +1,81 @@
+package io.onedev.server.git.command;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.StringTokenizer;
+
+import org.eclipse.jgit.util.QuotedString;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+
+import io.onedev.commons.utils.command.Commandline;
+import io.onedev.commons.utils.command.LineConsumer;
+
+public class ListFileChangesCommand extends GitCommand<Collection<FileChange>> {
+
+ private static final Logger logger = LoggerFactory.getLogger(ListFileChangesCommand.class);
+
+ private String fromRev;
+
+ private String toRev;
+
+ public ListFileChangesCommand(File gitDir) {
+ super(gitDir);
+ }
+
+ public ListFileChangesCommand fromRev(String fromRev) {
+ this.fromRev = fromRev;
+ return this;
+ }
+
+ public ListFileChangesCommand toRev(String toRev) {
+ this.toRev = toRev;
+ return this;
+ }
+
+ @Override
+ public Collection<FileChange> call() {
+ Preconditions.checkNotNull(toRev, "toRev has to be specified.");
+ Preconditions.checkNotNull(fromRev, "fromRev has to be specified.");
+
+ List<FileChange> fileChanges = new ArrayList<>();
+
+ Commandline cmd = cmd();
+
+ cmd.addArgs("diff", "--name-status", "--no-renames", fromRev + ".." + toRev);
+
+ cmd.execute(new LineConsumer() {
+
+ @Override
+ public void consume(String line) {
+ StringTokenizer tokenizer = new StringTokenizer(line, "\t");
+ String statusCode = tokenizer.nextToken();
+ if (statusCode.equals("M") || statusCode.equals("T")) {
+ String path = QuotedString.GIT_PATH.dequote(tokenizer.nextToken("\t"));
+ fileChanges.add(new FileChange(path, path, -1, -1));
+ } else if (statusCode.equals("D")) {
+ String oldPath = QuotedString.GIT_PATH.dequote(tokenizer.nextToken("\t"));
+ fileChanges.add(new FileChange(oldPath, null, -1, -1));
+ } else if (statusCode.equals("A")) {
+ String newPath = QuotedString.GIT_PATH.dequote(tokenizer.nextToken("\t"));
+ fileChanges.add(new FileChange(null, newPath, -1, -1));
+ }
+ }
+
+ }, new LineConsumer() {
+
+ @Override
+ public void consume(String line) {
+ logger.error(line);
+ }
+
+ }).checkReturnCode();
+
+ return fileChanges;
+ }
+
+}
diff --git a/server-core/src/main/java/io/onedev/server/git/command/ListFilesCommand.java b/server-core/src/main/java/io/onedev/server/git/command/ListFilesCommand.java
new file mode 100644
index 0000000000..f57f3786c5
--- /dev/null
+++ b/server-core/src/main/java/io/onedev/server/git/command/ListFilesCommand.java
@@ -0,0 +1,62 @@
+package io.onedev.server.git.command;
+
+import java.io.File;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.eclipse.jgit.util.QuotedString;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+
+import io.onedev.commons.utils.command.Commandline;
+import io.onedev.commons.utils.command.LineConsumer;
+
+public class ListFilesCommand extends GitCommand<Collection<String>> {
+
+ private static final Logger logger = LoggerFactory.getLogger(ListFilesCommand.class);
+
+ private String revision;
+
+ public ListFilesCommand(File gitDir) {
+ super(gitDir);
+ }
+
+ public ListFilesCommand revision(String revision) {
+ this.revision = revision;
+ return this;
+ }
+
+ @Override
+ public Collection<String> call() {
+ Preconditions.checkNotNull(revision, "revision has to be specified.");
+
+ Set<String> files = new HashSet<String>();
+
+ Commandline cmd = cmd();
+
+ cmd.addArgs("ls-tree", "--name-only", "-r", revision);
+
+ cmd.execute(new LineConsumer() {
+
+ @Override
+ public void consume(String line) {
+ if (line.trim().length() != 0)
+ files.add(QuotedString.GIT_PATH.dequote(line));
+ }
+
+ }, new LineConsumer() {
+
+ @Override
+ public void consume(String line) {
+ logger.error(line);
+ }
+
+ }).checkReturnCode();
+
+ return files;
+ }
+
+}
diff --git a/server-core/src/main/java/io/onedev/server/infomanager/DefaultCommitInfoManager.java b/server-core/src/main/java/io/onedev/server/infomanager/DefaultCommitInfoManager.java
index 95b5f2a130..20b92008d1 100644
--- a/server-core/src/main/java/io/onedev/server/infomanager/DefaultCommitInfoManager.java
+++ b/server-core/src/main/java/io/onedev/server/infomanager/DefaultCommitInfoManager.java
@@ -55,6 +55,8 @@ import io.onedev.server.git.GitContributor;
import io.onedev.server.git.GitUtils;
import io.onedev.server.git.command.FileChange;
import io.onedev.server.git.command.GitCommit;
+import io.onedev.server.git.command.ListFileChangesCommand;
+import io.onedev.server.git.command.ListFilesCommand;
import io.onedev.server.git.command.ListNumStatsCommand;
import io.onedev.server.git.command.LogCommand;
import io.onedev.server.git.command.RevListCommand;
@@ -89,14 +91,12 @@ public class DefaultCommitInfoManager extends AbstractMultiEnvironmentManager im
private static final Logger logger = LoggerFactory.getLogger(DefaultCommitInfoManager.class);
- private static final int INFO_VERSION = 13;
+ private static final int INFO_VERSION = 14;
private static final long LOG_FILE_SIZE = 256*1024;
private static final int COLLECT_BATCH_SIZE = 10000;
- private static final int MAX_COLLECTING_FILES = 50000;
-
private static final int MAX_HISTORY_PATHS = 100;
private static final int MAX_COMMIT_FILES = 100;
@@ -137,6 +137,8 @@ public class DefaultCommitInfoManager extends AbstractMultiEnvironmentManager im
private static final ByteIterable LAST_COMMIT_OF_CONTRIBS_KEY = new StringByteIterable("lastCommitOfContribs");
+ private static final ByteIterable LAST_COMMIT_OF_FILES_KEY = new StringByteIterable("lastCommitOfFiles");
+
private static final ByteIterable LINE_STATS_KEY = new StringByteIterable("lineStats");
private static final ByteIterable USERS_KEY = new StringByteIterable("users");
@@ -252,13 +254,6 @@ public class DefaultCommitInfoManager extends AbstractMultiEnvironmentManager im
else
users = new HashSet<>();
- Map<String, Long> files;
- byte[] fileBytes = readBytes(defaultStore, txn, FILES_KEY);
- if (fileBytes != null)
- files = (Map<String, Long>) SerializationUtils.deserialize(fileBytes);
- else
- files = new HashMap<>();
-
new ElementPumper<GitCommit>() {
@Override
@@ -337,11 +332,6 @@ public class DefaultCommitInfoManager extends AbstractMultiEnvironmentManager im
writeCommits(fixCommitsStore, txn, issueKey, fixingCommits);
}
- if (currentCommit.getCommitDate() != null) {
- for (String file: currentCommit.getChangedFiles())
- files.put(file, currentCommit.getCommitDate().getTime());
- }
-
if (currentCommit.getCommitter() != null)
users.add(new NameAndEmail(currentCommit.getCommitter()));
@@ -434,16 +424,6 @@ public class DefaultCommitInfoManager extends AbstractMultiEnvironmentManager im
defaultStore.put(txn, USERS_KEY, new ArrayByteIterable(userBytes));
usersCache.remove(project.getId());
- if (files.size() > MAX_COLLECTING_FILES) {
- List<String> fileList = new ArrayList<>(files.keySet());
- fileList.sort((file1, file2)->files.get(file1).compareTo(files.get(file2)));
- for (int i=0; i<fileList.size() - MAX_COLLECTING_FILES; i++)
- files.remove(fileList.get(i));
- }
- fileBytes = SerializationUtils.serialize((Serializable) files);
- defaultStore.put(txn, FILES_KEY, new ArrayByteIterable(fileBytes));
- filesCache.remove(project.getId());
-
for (Map.Entry<Long, Integer> entry: commitCountCache.entrySet())
writeInt(commitCountsStore, txn, new LongByteIterable(entry.getKey()), entry.getValue());
@@ -458,6 +438,7 @@ public class DefaultCommitInfoManager extends AbstractMultiEnvironmentManager im
if (GitUtils.branch2ref(project.getDefaultBranch()).equals(refName)) {
collectLineStats(project, commitId);
collectContribs(project, commitId);
+ collectFiles(project, commitId);
}
logger.debug("Collected commit information (project: {}, ref: {})", project.getPath(), refName);
@@ -573,6 +554,85 @@ public class DefaultCommitInfoManager extends AbstractMultiEnvironmentManager im
});
}
+ private void collectFiles(Project project, ObjectId commitId) {
+ Environment env = getEnv(project.getId().toString());
+ Store defaultStore = getStore(env, DEFAULT_STORE);
+
+ Repository repository = project.getRepository();
+
+ ObjectId lastCommitId = env.computeInTransaction(new TransactionalComputable<ObjectId>() {
+
+ @Override
+ public ObjectId compute(Transaction txn) {
+ ObjectId lastCommitId;
+ byte[] lastCommitBytes = readBytes(defaultStore, txn, LAST_COMMIT_OF_FILES_KEY);
+ if (lastCommitBytes != null) {
+ lastCommitId = ObjectId.fromRaw(lastCommitBytes);
+ try {
+ if (!repository.getObjectDatabase().has(lastCommitId))
+ lastCommitId = null;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ lastCommitId = null;
+ }
+ return lastCommitId;
+ }
+ });
+
+ if (lastCommitId == null) {
+ env.executeInTransaction(new TransactionalExecutable() {
+
+ @Override
+ public void execute(Transaction txn) {
+ ListFilesCommand command = new ListFilesCommand(project.getGitDir());
+ Collection<String> files = command.revision(commitId.name()).call();
+
+ byte[] bytesOfFiles = SerializationUtils.serialize((Serializable) files);
+ defaultStore.put(txn, FILES_KEY, new ArrayByteIterable(bytesOfFiles));
+ defaultStore.put(txn, LAST_COMMIT_OF_FILES_KEY, new CommitByteIterable(commitId));
+ filesCache.remove(project.getId());
+ }
+
+ });
+ } else {
+ env.executeInTransaction(new TransactionalExecutable() {
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void execute(Transaction txn) {
+ Collection<String> files;
+ byte[] bytesOfFiles = readBytes(defaultStore, txn, FILES_KEY);
+ if (bytesOfFiles != null)
+ files = (Set<String>) SerializationUtils.deserialize(bytesOfFiles);
+ else
+ files = new HashSet<>();
+
+ boolean filesChanged = false;
+ ListFileChangesCommand command = new ListFileChangesCommand(project.getGitDir());
+ for (FileChange change: command.fromRev(lastCommitId.name()).toRev(commitId.name()).call()) {
+ if (change.getOldPath() == null && change.getNewPath() != null) {
+ files.add(change.getNewPath());
+ filesChanged = true;
+ } else if (change.getOldPath() != null && change.getNewPath() == null) {
+ files.remove(change.getOldPath());
+ filesChanged = true;
+ }
+ }
+
+ if (filesChanged) {
+ bytesOfFiles = SerializationUtils.serialize((Serializable) files);
+ defaultStore.put(txn, FILES_KEY, new ArrayByteIterable(bytesOfFiles));
+ defaultStore.put(txn, LAST_COMMIT_OF_FILES_KEY, new CommitByteIterable(commitId));
+ filesCache.remove(project.getId());
+ }
+ }
+
+ });
+ }
+ }
+
private void collectLineStats(Project project, ObjectId commitId) {
Environment env = getEnv(project.getId().toString());
Store defaultStore = getStore(env, DEFAULT_STORE);
@@ -672,7 +732,6 @@ public class DefaultCommitInfoManager extends AbstractMultiEnvironmentManager im
}
});
-
}
}
@@ -818,7 +877,7 @@ public class DefaultCommitInfoManager extends AbstractMultiEnvironmentManager im
List<String> files = filesCache.get(project.getId());
if (files == null) {
Environment env = getEnv(project.getId().toString());
- final Store store = getStore(env, DEFAULT_STORE);
+ Store store = getStore(env, DEFAULT_STORE);
files = env.computeInReadonlyTransaction(new TransactionalComputable<List<String>>() {
@@ -827,13 +886,11 @@ public class DefaultCommitInfoManager extends AbstractMultiEnvironmentManager im
public List<String> compute(Transaction txn) {
byte[] bytes = readBytes(store, txn, FILES_KEY);
if (bytes != null) {
- List<String> files = new ArrayList<>(
- ((Map<String, Long>)SerializationUtils.deserialize(bytes)).keySet());
+ List<String> files = new ArrayList<>((Collection<String>)SerializationUtils.deserialize(bytes));
Map<String, List<String>> segmentsMap = new HashMap<>();
Splitter splitter = Splitter.on("/");
- for (String file: files) {
+ for (String file: files)
segmentsMap.put(file, splitter.splitToList(file));
- }
files.sort(new Comparator<String>() {
@Override