The Unified and Autonomous Development Platform
chore: Performance improvements
Robin Shenc279a41389df
diff --git a/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultProjectManager.java b/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultProjectManager.javaindex 85e52600b8..9de6b886b8 100644--- a/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultProjectManager.java+++ b/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultProjectManager.java@@ -844,7 +844,7 @@ public class DefaultProjectManager extends BaseEntityManager<Project>var projectId = Long.valueOf(projectDir.getName());var project = projects.get(projectId);if (project != null) {- logger.debug("Checking project (path: {}, id: {})...", project.getPath(), projectId);+ logger.debug("Checking project (path: {})...", project.getPath());checkGitDir(projectId);HookUtils.checkHooks(getGitDir(projectId));checkGitConfig(projectId, project.getGitPackConfig());diff --git a/server-core/src/main/java/io/onedev/server/search/code/DefaultCodeIndexManager.java b/server-core/src/main/java/io/onedev/server/search/code/DefaultCodeIndexManager.javaindex 49564084a9..aeac5afd8b 100644--- a/server-core/src/main/java/io/onedev/server/search/code/DefaultCodeIndexManager.java+++ b/server-core/src/main/java/io/onedev/server/search/code/DefaultCodeIndexManager.java@@ -66,9 +66,9 @@ public class DefaultCodeIndexManager implements CodeIndexManager, Serializable {private static final Logger logger = LoggerFactory.getLogger(DefaultCodeIndexManager.class);- private static final int UI_INDEXING_PRIORITY = 10;+ private static final int UI_PRIORITY = 10;- private static final int BACKEND_INDEXING_PRIORITY = 90;+ private static final int BACKEND_PRIORITY = 150;private static final int DATA_VERSION = 7;@@ -280,7 +280,7 @@ public class DefaultCodeIndexManager implements CodeIndexManager, Serializable {}private BatchWorker getBatchWorker(Long projectId) {- return new BatchWorker("project-" + projectId + "-indexBlob", 1) {+ return new BatchWorker("project-" + projectId + "-index-commit", 1) {@Overridepublic void doWorks(List<Prioritized> works) {@@ -307,7 +307,7 @@ public class DefaultCodeIndexManager implements CodeIndexManager, Serializable {IndexResult indexResult = index(projectManager.getRepository(project.getId()),commit, writer, searcher, PatternSet.parse(project.findCodeAnalysisPatterns()));writer.commit();- logger.debug("Commit indexed (project: {}, commit: {})", project.getPath(), commit.getName());+ logger.debug("Indexed commit (project: {}, commit: {})", project.getPath(), commit.getName());return indexResult;} catch (Exception e) {writer.rollback();@@ -383,7 +383,7 @@ public class DefaultCodeIndexManager implements CodeIndexManager, Serializable {// as many tags might be pushed all at once when the repository is importedif (event.getRefName().startsWith(Constants.R_HEADS)&& !event.getNewCommitId().equals(ObjectId.zeroId())) {- IndexWork work = new IndexWork(BACKEND_INDEXING_PRIORITY, event.getNewCommitId());+ IndexWork work = new IndexWork(BACKEND_PRIORITY, event.getNewCommitId());batchWorkManager.submit(getBatchWorker(event.getProject().getId()), work);}}@@ -415,9 +415,9 @@ public class DefaultCodeIndexManager implements CodeIndexManager, Serializable {public void indexAsync(Long projectId, ObjectId commitId) {int priority;if (RequestCycle.get() != null)- priority = UI_INDEXING_PRIORITY;+ priority = UI_PRIORITY;else- priority = BACKEND_INDEXING_PRIORITY;+ priority = BACKEND_PRIORITY;projectManager.runOnActiveServer(projectId, new ClusterTask<Void>() {private static final long serialVersionUID = 1L;diff --git a/server-core/src/main/java/io/onedev/server/search/entitytext/DefaultCodeCommentTextManager.java b/server-core/src/main/java/io/onedev/server/search/entitytext/DefaultCodeCommentTextManager.javaindex 7bedf9f04f..e79b9c637a 100644--- a/server-core/src/main/java/io/onedev/server/search/entitytext/DefaultCodeCommentTextManager.java+++ b/server-core/src/main/java/io/onedev/server/search/entitytext/DefaultCodeCommentTextManager.java@@ -85,7 +85,7 @@ public class DefaultCodeCommentTextManager extends EntityTextManager<CodeComment@Transactional@Listenpublic void on(CodeCommentTouched event) {- requestToIndex(event.getProject().getId());+ requestToIndex(event.getProject().getId(), UPDATE_PRIORITY);}@Overridediff --git a/server-core/src/main/java/io/onedev/server/search/entitytext/DefaultIssueTextManager.java b/server-core/src/main/java/io/onedev/server/search/entitytext/DefaultIssueTextManager.javaindex 681c194c52..482d896f39 100644--- a/server-core/src/main/java/io/onedev/server/search/entitytext/DefaultIssueTextManager.java+++ b/server-core/src/main/java/io/onedev/server/search/entitytext/DefaultIssueTextManager.java@@ -110,7 +110,7 @@ public class DefaultIssueTextManager extends EntityTextManager<Issue> implements@Transactional@Listenpublic void on(IssuesTouched event) {- requestToIndex(event.getProject().getId());+ requestToIndex(event.getProject().getId(), UPDATE_PRIORITY);}@Nullablediff --git a/server-core/src/main/java/io/onedev/server/search/entitytext/DefaultPullRequestTextManager.java b/server-core/src/main/java/io/onedev/server/search/entitytext/DefaultPullRequestTextManager.javaindex 44bcc5370b..bed12df7a2 100644--- a/server-core/src/main/java/io/onedev/server/search/entitytext/DefaultPullRequestTextManager.java+++ b/server-core/src/main/java/io/onedev/server/search/entitytext/DefaultPullRequestTextManager.java@@ -88,7 +88,7 @@ public class DefaultPullRequestTextManager extends EntityTextManager<PullRequest@Transactional@Listenpublic void on(PullRequestTouched event) {- requestToIndex(event.getProject().getId());+ requestToIndex(event.getProject().getId(), UPDATE_PRIORITY);}@Overridediff --git a/server-core/src/main/java/io/onedev/server/search/entitytext/EntityTextManager.java b/server-core/src/main/java/io/onedev/server/search/entitytext/EntityTextManager.javaindex b2b3936c8a..07c798535e 100644--- a/server-core/src/main/java/io/onedev/server/search/entitytext/EntityTextManager.java+++ b/server-core/src/main/java/io/onedev/server/search/entitytext/EntityTextManager.java@@ -108,7 +108,9 @@ public abstract class EntityTextManager<T extends ProjectBelonging> implements Sprotected static final String FIELD_PROJECT_ID = "projectId";- private static final int PRIORITY = 100;+ protected static final int UPDATE_PRIORITY = 100;++ private static final int CHECK_PRIORITY = 200;private static final int BATCH_SIZE = 5000;@@ -207,7 +209,7 @@ public abstract class EntityTextManager<T extends ProjectBelonging> implements S@Listenpublic void on(SystemStarted event) {for (var projectId: projectManager.getActiveIds())- requestToIndex(projectId);+ requestToIndex(projectId, CHECK_PRIORITY);}@Listen@@ -240,7 +242,7 @@ public abstract class EntityTextManager<T extends ProjectBelonging> implements S@Listenpublic void on(ActiveServerChanged event) {for(var projectId: event.getProjectIds())- requestToIndex(projectId);+ requestToIndex(projectId, CHECK_PRIORITY);}protected <R> R callWithSearcher(Function<IndexSearcher, R> func) {@@ -261,14 +263,14 @@ public abstract class EntityTextManager<T extends ProjectBelonging> implements S}}- protected void requestToIndex(Long projectId) {- var batchWorker = new BatchWorker("project-" + projectId + "-indexText-" + entityClass.getSimpleName()) {+ protected void requestToIndex(Long projectId, int priority) {+ var batchWorker = new BatchWorker("project-" + projectId + "-index-entity-" + entityClass.getSimpleName()) {@Overridepublic void doWorks(List<Prioritized> works) {String entityName = WordUtils.uncamel(entityClass.getSimpleName()).toLowerCase();String projectPath = projectManager.findFacadeById(projectId).getPath();- logger.debug("Indexing {} (project: {})", entityName, projectPath);+ logger.debug("Indexing {} (project: {})...", entityName, projectPath);var touchInfo = callWithSearcher(searcher -> {var touchId = 0L;@@ -325,7 +327,7 @@ public abstract class EntityTextManager<T extends ProjectBelonging> implements S}};- batchWorkManager.submit(batchWorker, new IndexWork(PRIORITY));+ batchWorkManager.submit(batchWorker, new IndexWork(priority));}private Query parse(String queryString) {diff --git a/server-core/src/main/java/io/onedev/server/xodus/AbstractEnvironmentManager.java b/server-core/src/main/java/io/onedev/server/xodus/AbstractEnvironmentManager.javaindex d699eb5124..e4102b66e9 100644--- a/server-core/src/main/java/io/onedev/server/xodus/AbstractEnvironmentManager.java+++ b/server-core/src/main/java/io/onedev/server/xodus/AbstractEnvironmentManager.java@@ -51,7 +51,7 @@ public abstract class AbstractEnvironmentManager {if (versionFromFile != getEnvVersion()) {FileUtils.cleanDir(versionFile.getParentFile());FileUtils.writeFile(versionFile, String.valueOf(getEnvVersion()));- }+ }}protected void writeVersion(File envDir) {diff --git a/server-core/src/main/java/io/onedev/server/xodus/DefaultCommitInfoManager.java b/server-core/src/main/java/io/onedev/server/xodus/DefaultCommitInfoManager.javaindex 255c31f2f5..929e20bcab 100644--- a/server-core/src/main/java/io/onedev/server/xodus/DefaultCommitInfoManager.java+++ b/server-core/src/main/java/io/onedev/server/xodus/DefaultCommitInfoManager.java@@ -183,7 +183,9 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManagerprivate static final ByteIterable COMMIT_COUNT_KEY = new StringByteIterable("commitCount");- private static final int COLLECT_PRIORITY = 100;+ private static final int UPDATE_PRIORITY = 100;++ private static final int CHECK_PRIORITY = 200;private final BatchWorkManager batchWorkManager;@@ -1110,7 +1112,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager};}- private void collect(Long projectId) {+ private void collect(Long projectId, int priority) {List<CollectingWork> works = new ArrayList<>();try (RevWalk revWalk = new RevWalk(projectManager.getRepository(projectId))) {Collection<Ref> refs = new ArrayList<>();@@ -1127,7 +1129,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager}if (revObj instanceof RevCommit) {RevCommit commit = (RevCommit) revObj;- works.add(new CollectingWork(COLLECT_PRIORITY, commit.copy(),+ works.add(new CollectingWork(priority, commit.copy(),commit.getCommitTime(), ref.getName()));}}@@ -1150,9 +1152,10 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager@Sessional@Listenpublic void on(SystemStarted event) {+ logger.info("Caching code contribution info...");for (var projectId: projectManager.getActiveIds()) {checkVersion(getEnvDir(projectId.toString()));- collect(projectId);+ collect(projectId, CHECK_PRIORITY);Environment env = getEnv(projectId.toString());Store store = getStore(env, DEFAULT_STORE);@@ -1172,7 +1175,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManagerpublic void on(ActiveServerChanged event) {for (var projectId: event.getProjectIds()) {checkVersion(getEnvDir(projectId.toString()));- collect(projectId);+ collect(projectId, CHECK_PRIORITY);}}@@ -1186,7 +1189,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManagertry (RevWalk revWalk = new RevWalk(repository)) {RevCommit commit = GitUtils.parseCommit(revWalk, event.getNewCommitId());if (commit != null) {- CollectingWork work = new CollectingWork(COLLECT_PRIORITY, commit.copy(),+ CollectingWork work = new CollectingWork(UPDATE_PRIORITY, commit.copy(),commit.getCommitTime(), event.getRefName());batchWorkManager.submit(getBatchWorker(event.getProject().getId()), work);}diff --git a/server-core/src/main/java/io/onedev/server/xodus/DefaultIssueInfoManager.java b/server-core/src/main/java/io/onedev/server/xodus/DefaultIssueInfoManager.javaindex 1cc18f7963..17f5ee7c4e 100644--- a/server-core/src/main/java/io/onedev/server/xodus/DefaultIssueInfoManager.java+++ b/server-core/src/main/java/io/onedev/server/xodus/DefaultIssueInfoManager.java@@ -72,7 +72,9 @@ public class DefaultIssueInfoManager extends AbstractEnvironmentManagerprivate static final ByteIterable LAST_ISSUE_CHANGE_KEY = new StringByteIterable("lastIssueChange");- private static final int PRIORITY = 100;+ private static final int UPDATE_PRIORITY = 100;++ private static final int CHECK_PRIORITY = 200;private final BatchWorkManager batchWorkManager;@@ -96,7 +98,7 @@ public class DefaultIssueInfoManager extends AbstractEnvironmentManager}private BatchWorker getBatchWorker(Long projectId) {- return new BatchWorker("project-" + projectId + "-collectIssueInfo") {+ return new BatchWorker("project-" + projectId + "-collect-issue-info") {@Overridepublic void doWorks(List<Prioritized> works) {@@ -109,7 +111,8 @@ public class DefaultIssueInfoManager extends AbstractEnvironmentManager@Sessionalprotected boolean collect(Long projectId) {- logger.debug("Collecting issue info...");+ var projectPath = projectManager.findFacadeById(projectId).getPath();+ logger.debug("Collecting issue info (project: {})...", projectPath);Environment env = getEnv(projectId.toString());Store defaultStore = getStore(env, DEFAULT_STORE);@@ -178,7 +181,7 @@ public class DefaultIssueInfoManager extends AbstractEnvironmentManagerdefaultStore.put(txn, LAST_ISSUE_CHANGE_KEY, new LongByteIterable(lastChange.getId()));});- logger.debug("Collected issue info");+ logger.debug("Collected issue info (project: {})", projectPath);return unprocessedIssues.size() == BATCH_SIZE;}@@ -220,25 +223,25 @@ public class DefaultIssueInfoManager extends AbstractEnvironmentManager@Sessional@Listenpublic void on(IssueOpened event) {- batchWorkManager.submit(getBatchWorker(event.getProject().getId()), new Prioritized(PRIORITY));+ batchWorkManager.submit(getBatchWorker(event.getProject().getId()), new Prioritized(UPDATE_PRIORITY));}@Sessional@Listenpublic void on(IssuesCopied event) {- batchWorkManager.submit(getBatchWorker(event.getProject().getId()), new Prioritized(PRIORITY));+ batchWorkManager.submit(getBatchWorker(event.getProject().getId()), new Prioritized(UPDATE_PRIORITY));}@Sessional@Listenpublic void on(IssuesImported event) {- batchWorkManager.submit(getBatchWorker(event.getProject().getId()), new Prioritized(PRIORITY));+ batchWorkManager.submit(getBatchWorker(event.getProject().getId()), new Prioritized(UPDATE_PRIORITY));}@Sessional@Listenpublic void on(IssueChanged event) {- batchWorkManager.submit(getBatchWorker(event.getProject().getId()), new Prioritized(PRIORITY));+ batchWorkManager.submit(getBatchWorker(event.getProject().getId()), new Prioritized(UPDATE_PRIORITY));}@Sessional@@ -274,7 +277,7 @@ public class DefaultIssueInfoManager extends AbstractEnvironmentManager});- batchWorkManager.submit(getBatchWorker(event.getProject().getId()), new Prioritized(PRIORITY));+ batchWorkManager.submit(getBatchWorker(event.getProject().getId()), new Prioritized(UPDATE_PRIORITY));}@Sessional@@ -282,7 +285,7 @@ public class DefaultIssueInfoManager extends AbstractEnvironmentManagerpublic void on(SystemStarted event) {for (var projectId: projectManager.getActiveIds()) {checkVersion(getEnvDir(projectId.toString()));- batchWorkManager.submit(getBatchWorker(projectId), new Prioritized(PRIORITY));+ batchWorkManager.submit(getBatchWorker(projectId), new Prioritized(CHECK_PRIORITY));}}@@ -291,7 +294,7 @@ public class DefaultIssueInfoManager extends AbstractEnvironmentManagerpublic void on(ActiveServerChanged event) {for (var projectId: event.getProjectIds()) {checkVersion(getEnvDir(projectId.toString()));- batchWorkManager.submit(getBatchWorker(projectId), new Prioritized(PRIORITY));+ batchWorkManager.submit(getBatchWorker(projectId), new Prioritized(CHECK_PRIORITY));}}diff --git a/server-core/src/main/java/io/onedev/server/xodus/DefaultPullRequestInfoManager.java b/server-core/src/main/java/io/onedev/server/xodus/DefaultPullRequestInfoManager.javaindex c27f9696f2..052c44ac22 100644--- a/server-core/src/main/java/io/onedev/server/xodus/DefaultPullRequestInfoManager.java+++ b/server-core/src/main/java/io/onedev/server/xodus/DefaultPullRequestInfoManager.java@@ -55,7 +55,9 @@ public class DefaultPullRequestInfoManager extends AbstractEnvironmentManagerprivate static final ByteIterable LAST_PULL_REQUEST_UPDATE_KEY = new StringByteIterable("lastPullRequestUpdate");- private static final int PRIORITY = 100;+ private static final int UPDATE_PRIORITY = 100;++ private static final int CHECK_PRIORITY = 200;private final BatchWorkManager batchWorkManager;@@ -94,7 +96,8 @@ public class DefaultPullRequestInfoManager extends AbstractEnvironmentManager@Sessionalprotected boolean collect(Long projectId) {- logger.debug("Collecting pull request info (project id: {})...", projectId);+ var projectPath = projectManager.findFacadeById(projectId).getPath();+ logger.debug("Collecting pull request info (project: {})...", projectPath);Environment env = getEnv(projectId.toString());Store defaultStore = getStore(env, DEFAULT_STORE);@@ -121,7 +124,7 @@ public class DefaultPullRequestInfoManager extends AbstractEnvironmentManagerif (lastUpdate != null)defaultStore.put(txn, LAST_PULL_REQUEST_UPDATE_KEY, new LongByteIterable(lastUpdate.getId()));});- logger.debug("Collected pull request info (project id: {})", projectId);+ logger.debug("Collected pull request info (project: {})", projectPath);return unprocessedPullRequestUpdates.size() == BATCH_SIZE;}@@ -148,13 +151,13 @@ public class DefaultPullRequestInfoManager extends AbstractEnvironmentManager@Sessional@Listenpublic void on(PullRequestOpened event) {- batchWorkManager.submit(getBatchWorker(event.getProject().getId()), new Prioritized(PRIORITY));+ batchWorkManager.submit(getBatchWorker(event.getProject().getId()), new Prioritized(UPDATE_PRIORITY));}@Sessional@Listenpublic void on(PullRequestUpdated event) {- batchWorkManager.submit(getBatchWorker(event.getProject().getId()), new Prioritized(PRIORITY));+ batchWorkManager.submit(getBatchWorker(event.getProject().getId()), new Prioritized(UPDATE_PRIORITY));}@Sessional@@ -162,7 +165,7 @@ public class DefaultPullRequestInfoManager extends AbstractEnvironmentManagerpublic void on(SystemStarted event) {for (var projectId: projectManager.getActiveIds()) {checkVersion(getEnvDir(projectId.toString()));- batchWorkManager.submit(getBatchWorker(projectId), new Prioritized(PRIORITY));+ batchWorkManager.submit(getBatchWorker(projectId), new Prioritized(CHECK_PRIORITY));}}@@ -171,7 +174,7 @@ public class DefaultPullRequestInfoManager extends AbstractEnvironmentManagerpublic void on(ActiveServerChanged event) {for (var projectId: event.getProjectIds()) {checkVersion(getEnvDir(projectId.toString()));- batchWorkManager.submit(getBatchWorker(projectId), new Prioritized(PRIORITY));+ batchWorkManager.submit(getBatchWorker(projectId), new Prioritized(CHECK_PRIORITY));}}diff --git a/server-ee b/server-eeindex 32e2730088..896f3a3b23 160000--- a/server-ee+++ b/server-ee@@ -1 +1 @@-Subproject commit 32e27300885177aceba96d80812e71e0867e01da+Subproject commit 896f3a3b23b21c2b3625dbe920bd84ddae565010