Skip to content

The Unified and Autonomous Development Platform

chore: Performance improvements

Robin Shenc279a41389df
diff --git a/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultProjectManager.java b/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultProjectManager.java
index 85e52600b8..9de6b886b8 100644
--- a/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultProjectManager.java
+++ b/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultProjectManager.java
@@ -844,7 +844,7 @@ public class DefaultProjectManager extends BaseEntityManager<Project>
var projectId = Long.valueOf(projectDir.getName());
var project = projects.get(projectId);
if (project != null) {
- logger.debug("Checking project (path: {}, id: {})...", project.getPath(), projectId);
+ logger.debug("Checking project (path: {})...", project.getPath());
checkGitDir(projectId);
HookUtils.checkHooks(getGitDir(projectId));
checkGitConfig(projectId, project.getGitPackConfig());
diff --git a/server-core/src/main/java/io/onedev/server/search/code/DefaultCodeIndexManager.java b/server-core/src/main/java/io/onedev/server/search/code/DefaultCodeIndexManager.java
index 49564084a9..aeac5afd8b 100644
--- a/server-core/src/main/java/io/onedev/server/search/code/DefaultCodeIndexManager.java
+++ b/server-core/src/main/java/io/onedev/server/search/code/DefaultCodeIndexManager.java
@@ -66,9 +66,9 @@ public class DefaultCodeIndexManager implements CodeIndexManager, Serializable {
private static final Logger logger = LoggerFactory.getLogger(DefaultCodeIndexManager.class);
- private static final int UI_INDEXING_PRIORITY = 10;
+ private static final int UI_PRIORITY = 10;
- private static final int BACKEND_INDEXING_PRIORITY = 90;
+ private static final int BACKEND_PRIORITY = 150;
private static final int DATA_VERSION = 7;
@@ -280,7 +280,7 @@ public class DefaultCodeIndexManager implements CodeIndexManager, Serializable {
}
private BatchWorker getBatchWorker(Long projectId) {
- return new BatchWorker("project-" + projectId + "-indexBlob", 1) {
+ return new BatchWorker("project-" + projectId + "-index-commit", 1) {
@Override
public void doWorks(List<Prioritized> works) {
@@ -307,7 +307,7 @@ public class DefaultCodeIndexManager implements CodeIndexManager, Serializable {
IndexResult indexResult = index(projectManager.getRepository(project.getId()),
commit, writer, searcher, PatternSet.parse(project.findCodeAnalysisPatterns()));
writer.commit();
- logger.debug("Commit indexed (project: {}, commit: {})", project.getPath(), commit.getName());
+ logger.debug("Indexed commit (project: {}, commit: {})", project.getPath(), commit.getName());
return indexResult;
} catch (Exception e) {
writer.rollback();
@@ -383,7 +383,7 @@ public class DefaultCodeIndexManager implements CodeIndexManager, Serializable {
// as many tags might be pushed all at once when the repository is imported
if (event.getRefName().startsWith(Constants.R_HEADS)
&& !event.getNewCommitId().equals(ObjectId.zeroId())) {
- IndexWork work = new IndexWork(BACKEND_INDEXING_PRIORITY, event.getNewCommitId());
+ IndexWork work = new IndexWork(BACKEND_PRIORITY, event.getNewCommitId());
batchWorkManager.submit(getBatchWorker(event.getProject().getId()), work);
}
}
@@ -415,9 +415,9 @@ public class DefaultCodeIndexManager implements CodeIndexManager, Serializable {
public void indexAsync(Long projectId, ObjectId commitId) {
int priority;
if (RequestCycle.get() != null)
- priority = UI_INDEXING_PRIORITY;
+ priority = UI_PRIORITY;
else
- priority = BACKEND_INDEXING_PRIORITY;
+ priority = BACKEND_PRIORITY;
projectManager.runOnActiveServer(projectId, new ClusterTask<Void>() {
private static final long serialVersionUID = 1L;
diff --git a/server-core/src/main/java/io/onedev/server/search/entitytext/DefaultCodeCommentTextManager.java b/server-core/src/main/java/io/onedev/server/search/entitytext/DefaultCodeCommentTextManager.java
index 7bedf9f04f..e79b9c637a 100644
--- a/server-core/src/main/java/io/onedev/server/search/entitytext/DefaultCodeCommentTextManager.java
+++ b/server-core/src/main/java/io/onedev/server/search/entitytext/DefaultCodeCommentTextManager.java
@@ -85,7 +85,7 @@ public class DefaultCodeCommentTextManager extends EntityTextManager<CodeComment
@Transactional
@Listen
public void on(CodeCommentTouched event) {
- requestToIndex(event.getProject().getId());
+ requestToIndex(event.getProject().getId(), UPDATE_PRIORITY);
}
@Override
diff --git a/server-core/src/main/java/io/onedev/server/search/entitytext/DefaultIssueTextManager.java b/server-core/src/main/java/io/onedev/server/search/entitytext/DefaultIssueTextManager.java
index 681c194c52..482d896f39 100644
--- a/server-core/src/main/java/io/onedev/server/search/entitytext/DefaultIssueTextManager.java
+++ b/server-core/src/main/java/io/onedev/server/search/entitytext/DefaultIssueTextManager.java
@@ -110,7 +110,7 @@ public class DefaultIssueTextManager extends EntityTextManager<Issue> implements
@Transactional
@Listen
public void on(IssuesTouched event) {
- requestToIndex(event.getProject().getId());
+ requestToIndex(event.getProject().getId(), UPDATE_PRIORITY);
}
@Nullable
diff --git a/server-core/src/main/java/io/onedev/server/search/entitytext/DefaultPullRequestTextManager.java b/server-core/src/main/java/io/onedev/server/search/entitytext/DefaultPullRequestTextManager.java
index 44bcc5370b..bed12df7a2 100644
--- a/server-core/src/main/java/io/onedev/server/search/entitytext/DefaultPullRequestTextManager.java
+++ b/server-core/src/main/java/io/onedev/server/search/entitytext/DefaultPullRequestTextManager.java
@@ -88,7 +88,7 @@ public class DefaultPullRequestTextManager extends EntityTextManager<PullRequest
@Transactional
@Listen
public void on(PullRequestTouched event) {
- requestToIndex(event.getProject().getId());
+ requestToIndex(event.getProject().getId(), UPDATE_PRIORITY);
}
@Override
diff --git a/server-core/src/main/java/io/onedev/server/search/entitytext/EntityTextManager.java b/server-core/src/main/java/io/onedev/server/search/entitytext/EntityTextManager.java
index b2b3936c8a..07c798535e 100644
--- a/server-core/src/main/java/io/onedev/server/search/entitytext/EntityTextManager.java
+++ b/server-core/src/main/java/io/onedev/server/search/entitytext/EntityTextManager.java
@@ -108,7 +108,9 @@ public abstract class EntityTextManager<T extends ProjectBelonging> implements S
protected static final String FIELD_PROJECT_ID = "projectId";
- private static final int PRIORITY = 100;
+ protected static final int UPDATE_PRIORITY = 100;
+
+ private static final int CHECK_PRIORITY = 200;
private static final int BATCH_SIZE = 5000;
@@ -207,7 +209,7 @@ public abstract class EntityTextManager<T extends ProjectBelonging> implements S
@Listen
public void on(SystemStarted event) {
for (var projectId: projectManager.getActiveIds())
- requestToIndex(projectId);
+ requestToIndex(projectId, CHECK_PRIORITY);
}
@Listen
@@ -240,7 +242,7 @@ public abstract class EntityTextManager<T extends ProjectBelonging> implements S
@Listen
public void on(ActiveServerChanged event) {
for(var projectId: event.getProjectIds())
- requestToIndex(projectId);
+ requestToIndex(projectId, CHECK_PRIORITY);
}
protected <R> R callWithSearcher(Function<IndexSearcher, R> func) {
@@ -261,14 +263,14 @@ public abstract class EntityTextManager<T extends ProjectBelonging> implements S
}
}
- protected void requestToIndex(Long projectId) {
- var batchWorker = new BatchWorker("project-" + projectId + "-indexText-" + entityClass.getSimpleName()) {
+ protected void requestToIndex(Long projectId, int priority) {
+ var batchWorker = new BatchWorker("project-" + projectId + "-index-entity-" + entityClass.getSimpleName()) {
@Override
public void doWorks(List<Prioritized> works) {
String entityName = WordUtils.uncamel(entityClass.getSimpleName()).toLowerCase();
String projectPath = projectManager.findFacadeById(projectId).getPath();
- logger.debug("Indexing {} (project: {})", entityName, projectPath);
+ logger.debug("Indexing {} (project: {})...", entityName, projectPath);
var touchInfo = callWithSearcher(searcher -> {
var touchId = 0L;
@@ -325,7 +327,7 @@ public abstract class EntityTextManager<T extends ProjectBelonging> implements S
}
};
- batchWorkManager.submit(batchWorker, new IndexWork(PRIORITY));
+ batchWorkManager.submit(batchWorker, new IndexWork(priority));
}
private Query parse(String queryString) {
diff --git a/server-core/src/main/java/io/onedev/server/xodus/AbstractEnvironmentManager.java b/server-core/src/main/java/io/onedev/server/xodus/AbstractEnvironmentManager.java
index d699eb5124..e4102b66e9 100644
--- a/server-core/src/main/java/io/onedev/server/xodus/AbstractEnvironmentManager.java
+++ b/server-core/src/main/java/io/onedev/server/xodus/AbstractEnvironmentManager.java
@@ -51,7 +51,7 @@ public abstract class AbstractEnvironmentManager {
if (versionFromFile != getEnvVersion()) {
FileUtils.cleanDir(versionFile.getParentFile());
FileUtils.writeFile(versionFile, String.valueOf(getEnvVersion()));
- }
+ }
}
protected void writeVersion(File envDir) {
diff --git a/server-core/src/main/java/io/onedev/server/xodus/DefaultCommitInfoManager.java b/server-core/src/main/java/io/onedev/server/xodus/DefaultCommitInfoManager.java
index 255c31f2f5..929e20bcab 100644
--- a/server-core/src/main/java/io/onedev/server/xodus/DefaultCommitInfoManager.java
+++ b/server-core/src/main/java/io/onedev/server/xodus/DefaultCommitInfoManager.java
@@ -183,7 +183,9 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager
private static final ByteIterable COMMIT_COUNT_KEY = new StringByteIterable("commitCount");
- private static final int COLLECT_PRIORITY = 100;
+ private static final int UPDATE_PRIORITY = 100;
+
+ private static final int CHECK_PRIORITY = 200;
private final BatchWorkManager batchWorkManager;
@@ -1110,7 +1112,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager
};
}
- private void collect(Long projectId) {
+ private void collect(Long projectId, int priority) {
List<CollectingWork> works = new ArrayList<>();
try (RevWalk revWalk = new RevWalk(projectManager.getRepository(projectId))) {
Collection<Ref> refs = new ArrayList<>();
@@ -1127,7 +1129,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager
}
if (revObj instanceof RevCommit) {
RevCommit commit = (RevCommit) revObj;
- works.add(new CollectingWork(COLLECT_PRIORITY, commit.copy(),
+ works.add(new CollectingWork(priority, commit.copy(),
commit.getCommitTime(), ref.getName()));
}
}
@@ -1150,9 +1152,10 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager
@Sessional
@Listen
public void on(SystemStarted event) {
+ logger.info("Caching code contribution info...");
for (var projectId: projectManager.getActiveIds()) {
checkVersion(getEnvDir(projectId.toString()));
- collect(projectId);
+ collect(projectId, CHECK_PRIORITY);
Environment env = getEnv(projectId.toString());
Store store = getStore(env, DEFAULT_STORE);
@@ -1172,7 +1175,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager
public void on(ActiveServerChanged event) {
for (var projectId: event.getProjectIds()) {
checkVersion(getEnvDir(projectId.toString()));
- collect(projectId);
+ collect(projectId, CHECK_PRIORITY);
}
}
@@ -1186,7 +1189,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager
try (RevWalk revWalk = new RevWalk(repository)) {
RevCommit commit = GitUtils.parseCommit(revWalk, event.getNewCommitId());
if (commit != null) {
- CollectingWork work = new CollectingWork(COLLECT_PRIORITY, commit.copy(),
+ CollectingWork work = new CollectingWork(UPDATE_PRIORITY, commit.copy(),
commit.getCommitTime(), event.getRefName());
batchWorkManager.submit(getBatchWorker(event.getProject().getId()), work);
}
diff --git a/server-core/src/main/java/io/onedev/server/xodus/DefaultIssueInfoManager.java b/server-core/src/main/java/io/onedev/server/xodus/DefaultIssueInfoManager.java
index 1cc18f7963..17f5ee7c4e 100644
--- a/server-core/src/main/java/io/onedev/server/xodus/DefaultIssueInfoManager.java
+++ b/server-core/src/main/java/io/onedev/server/xodus/DefaultIssueInfoManager.java
@@ -72,7 +72,9 @@ public class DefaultIssueInfoManager extends AbstractEnvironmentManager
private static final ByteIterable LAST_ISSUE_CHANGE_KEY = new StringByteIterable("lastIssueChange");
- private static final int PRIORITY = 100;
+ private static final int UPDATE_PRIORITY = 100;
+
+ private static final int CHECK_PRIORITY = 200;
private final BatchWorkManager batchWorkManager;
@@ -96,7 +98,7 @@ public class DefaultIssueInfoManager extends AbstractEnvironmentManager
}
private BatchWorker getBatchWorker(Long projectId) {
- return new BatchWorker("project-" + projectId + "-collectIssueInfo") {
+ return new BatchWorker("project-" + projectId + "-collect-issue-info") {
@Override
public void doWorks(List<Prioritized> works) {
@@ -109,7 +111,8 @@ public class DefaultIssueInfoManager extends AbstractEnvironmentManager
@Sessional
protected boolean collect(Long projectId) {
- logger.debug("Collecting issue info...");
+ var projectPath = projectManager.findFacadeById(projectId).getPath();
+ logger.debug("Collecting issue info (project: {})...", projectPath);
Environment env = getEnv(projectId.toString());
Store defaultStore = getStore(env, DEFAULT_STORE);
@@ -178,7 +181,7 @@ public class DefaultIssueInfoManager extends AbstractEnvironmentManager
defaultStore.put(txn, LAST_ISSUE_CHANGE_KEY, new LongByteIterable(lastChange.getId()));
});
- logger.debug("Collected issue info");
+ logger.debug("Collected issue info (project: {})", projectPath);
return unprocessedIssues.size() == BATCH_SIZE;
}
@@ -220,25 +223,25 @@ public class DefaultIssueInfoManager extends AbstractEnvironmentManager
@Sessional
@Listen
public void on(IssueOpened event) {
- batchWorkManager.submit(getBatchWorker(event.getProject().getId()), new Prioritized(PRIORITY));
+ batchWorkManager.submit(getBatchWorker(event.getProject().getId()), new Prioritized(UPDATE_PRIORITY));
}
@Sessional
@Listen
public void on(IssuesCopied event) {
- batchWorkManager.submit(getBatchWorker(event.getProject().getId()), new Prioritized(PRIORITY));
+ batchWorkManager.submit(getBatchWorker(event.getProject().getId()), new Prioritized(UPDATE_PRIORITY));
}
@Sessional
@Listen
public void on(IssuesImported event) {
- batchWorkManager.submit(getBatchWorker(event.getProject().getId()), new Prioritized(PRIORITY));
+ batchWorkManager.submit(getBatchWorker(event.getProject().getId()), new Prioritized(UPDATE_PRIORITY));
}
@Sessional
@Listen
public void on(IssueChanged event) {
- batchWorkManager.submit(getBatchWorker(event.getProject().getId()), new Prioritized(PRIORITY));
+ batchWorkManager.submit(getBatchWorker(event.getProject().getId()), new Prioritized(UPDATE_PRIORITY));
}
@Sessional
@@ -274,7 +277,7 @@ public class DefaultIssueInfoManager extends AbstractEnvironmentManager
});
- batchWorkManager.submit(getBatchWorker(event.getProject().getId()), new Prioritized(PRIORITY));
+ batchWorkManager.submit(getBatchWorker(event.getProject().getId()), new Prioritized(UPDATE_PRIORITY));
}
@Sessional
@@ -282,7 +285,7 @@ public class DefaultIssueInfoManager extends AbstractEnvironmentManager
public void on(SystemStarted event) {
for (var projectId: projectManager.getActiveIds()) {
checkVersion(getEnvDir(projectId.toString()));
- batchWorkManager.submit(getBatchWorker(projectId), new Prioritized(PRIORITY));
+ batchWorkManager.submit(getBatchWorker(projectId), new Prioritized(CHECK_PRIORITY));
}
}
@@ -291,7 +294,7 @@ public class DefaultIssueInfoManager extends AbstractEnvironmentManager
public void on(ActiveServerChanged event) {
for (var projectId: event.getProjectIds()) {
checkVersion(getEnvDir(projectId.toString()));
- batchWorkManager.submit(getBatchWorker(projectId), new Prioritized(PRIORITY));
+ batchWorkManager.submit(getBatchWorker(projectId), new Prioritized(CHECK_PRIORITY));
}
}
diff --git a/server-core/src/main/java/io/onedev/server/xodus/DefaultPullRequestInfoManager.java b/server-core/src/main/java/io/onedev/server/xodus/DefaultPullRequestInfoManager.java
index c27f9696f2..052c44ac22 100644
--- a/server-core/src/main/java/io/onedev/server/xodus/DefaultPullRequestInfoManager.java
+++ b/server-core/src/main/java/io/onedev/server/xodus/DefaultPullRequestInfoManager.java
@@ -55,7 +55,9 @@ public class DefaultPullRequestInfoManager extends AbstractEnvironmentManager
private static final ByteIterable LAST_PULL_REQUEST_UPDATE_KEY = new StringByteIterable("lastPullRequestUpdate");
- private static final int PRIORITY = 100;
+ private static final int UPDATE_PRIORITY = 100;
+
+ private static final int CHECK_PRIORITY = 200;
private final BatchWorkManager batchWorkManager;
@@ -94,7 +96,8 @@ public class DefaultPullRequestInfoManager extends AbstractEnvironmentManager
@Sessional
protected boolean collect(Long projectId) {
- logger.debug("Collecting pull request info (project id: {})...", projectId);
+ var projectPath = projectManager.findFacadeById(projectId).getPath();
+ logger.debug("Collecting pull request info (project: {})...", projectPath);
Environment env = getEnv(projectId.toString());
Store defaultStore = getStore(env, DEFAULT_STORE);
@@ -121,7 +124,7 @@ public class DefaultPullRequestInfoManager extends AbstractEnvironmentManager
if (lastUpdate != null)
defaultStore.put(txn, LAST_PULL_REQUEST_UPDATE_KEY, new LongByteIterable(lastUpdate.getId()));
});
- logger.debug("Collected pull request info (project id: {})", projectId);
+ logger.debug("Collected pull request info (project: {})", projectPath);
return unprocessedPullRequestUpdates.size() == BATCH_SIZE;
}
@@ -148,13 +151,13 @@ public class DefaultPullRequestInfoManager extends AbstractEnvironmentManager
@Sessional
@Listen
public void on(PullRequestOpened event) {
- batchWorkManager.submit(getBatchWorker(event.getProject().getId()), new Prioritized(PRIORITY));
+ batchWorkManager.submit(getBatchWorker(event.getProject().getId()), new Prioritized(UPDATE_PRIORITY));
}
@Sessional
@Listen
public void on(PullRequestUpdated event) {
- batchWorkManager.submit(getBatchWorker(event.getProject().getId()), new Prioritized(PRIORITY));
+ batchWorkManager.submit(getBatchWorker(event.getProject().getId()), new Prioritized(UPDATE_PRIORITY));
}
@Sessional
@@ -162,7 +165,7 @@ public class DefaultPullRequestInfoManager extends AbstractEnvironmentManager
public void on(SystemStarted event) {
for (var projectId: projectManager.getActiveIds()) {
checkVersion(getEnvDir(projectId.toString()));
- batchWorkManager.submit(getBatchWorker(projectId), new Prioritized(PRIORITY));
+ batchWorkManager.submit(getBatchWorker(projectId), new Prioritized(CHECK_PRIORITY));
}
}
@@ -171,7 +174,7 @@ public class DefaultPullRequestInfoManager extends AbstractEnvironmentManager
public void on(ActiveServerChanged event) {
for (var projectId: event.getProjectIds()) {
checkVersion(getEnvDir(projectId.toString()));
- batchWorkManager.submit(getBatchWorker(projectId), new Prioritized(PRIORITY));
+ batchWorkManager.submit(getBatchWorker(projectId), new Prioritized(CHECK_PRIORITY));
}
}
diff --git a/server-ee b/server-ee
index 32e2730088..896f3a3b23 160000
--- a/server-ee
+++ b/server-ee
@@ -1 +1 @@
-Subproject commit 32e27300885177aceba96d80812e71e0867e01da
+Subproject commit 896f3a3b23b21c2b3625dbe920bd84ddae565010