The Unified and Autonomous Development Platform
chore: Refactoring server side step
Robin Shen27c0204bb067
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/ServerSideStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/ServerSideStep.javaindex dd881108a7..d277fceef7 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/step/ServerSideStep.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/ServerSideStep.java@@ -36,11 +36,7 @@ public abstract class ServerSideStep extends Step {protected PatternSet getFiles() {return new PatternSet(new HashSet<>(), new HashSet<>());}-- public boolean requireCommitIndex() {- return false;- }-+public abstract ServerStepResult run(Long buildId, File inputDir, TaskLogger logger);@Overridediff --git a/server-core/src/main/java/io/onedev/server/git/GitUtils.java b/server-core/src/main/java/io/onedev/server/git/GitUtils.javaindex 5d546e94ae..bf61493805 100644--- a/server-core/src/main/java/io/onedev/server/git/GitUtils.java+++ b/server-core/src/main/java/io/onedev/server/git/GitUtils.java@@ -196,6 +196,24 @@ public class GitUtils {throw new RuntimeException(e);}}++ public static Collection<String> getBlobPaths(Repository repository, ObjectId commitId) {+ Collection<String> blobPaths = new HashSet<>();+ try (RevWalk revWalk = new RevWalk(repository);+ TreeWalk treeWalk = new TreeWalk(repository)) {+ RevCommit commit = revWalk.parseCommit(commitId);+ treeWalk.addTree(commit.getTree());+ treeWalk.setRecursive(true);+ treeWalk.setFilter(TreeFilter.ANY_DIFF);++ while (treeWalk.next()) {+ blobPaths.add(treeWalk.getPathString());+ }+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ return blobPaths;+ }@Nullablepublic static RevCommit getLastCommit(Repository repository) {diff --git a/server-core/src/main/java/io/onedev/server/job/DefaultJobManager.java b/server-core/src/main/java/io/onedev/server/job/DefaultJobManager.javaindex e030e8c316..a72b90e4ba 100644--- a/server-core/src/main/java/io/onedev/server/job/DefaultJobManager.java+++ b/server-core/src/main/java/io/onedev/server/job/DefaultJobManager.java@@ -149,7 +149,6 @@ import io.onedev.server.persistence.TransactionManager;import io.onedev.server.persistence.annotation.Sessional;import io.onedev.server.persistence.annotation.Transactional;import io.onedev.server.persistence.dao.Dao;-import io.onedev.server.search.code.CodeIndexManager;import io.onedev.server.security.CodePullAuthorizationSource;import io.onedev.server.security.SecurityUtils;import io.onedev.server.security.permission.AccessBuild;@@ -224,8 +223,6 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthorizprivate final Validator validator;private final ClusterManager clusterManager;-- private final CodeIndexManager codeIndexManager;private final GitService gitService;@@ -252,7 +249,7 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthorizListenerRegistry listenerRegistry, SettingManager settingManager, TransactionManager transactionManager,LogManager logManager, ExecutorService executorService, SessionManager sessionManager,BuildParamManager buildParamManager, ProjectManager projectManager, Validator validator,- TaskScheduler taskScheduler, ClusterManager clusterManager, CodeIndexManager codeIndexManager,+ TaskScheduler taskScheduler, ClusterManager clusterManager,PullRequestManager pullRequestManager, IssueManager issueManager, GitService gitService,SSLFactory sslFactory, Dao dao, BatchWorkManager batchWorkManager) {this.dao = dao;@@ -269,7 +266,6 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthorizthis.projectManager = projectManager;this.validator = validator;this.taskScheduler = taskScheduler;- this.codeIndexManager = codeIndexManager;this.clusterManager = clusterManager;this.pullRequestManager = pullRequestManager;this.issueManager = issueManager;@@ -1551,20 +1547,7 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthorizif (actions != null) {ServerSideFacade serverSideFacade = (ServerSideFacade) LeafFacade.of(actions, stepPosition);var serverSideStep = (ServerSideStep) serverSideFacade.getStep();- var transformedServerSideStep = new EditableStringTransformer(t -> replacePlaceholders(t, placeholderValues)).transformProperties(serverSideStep, Interpolative.class);-- if (transformedServerSideStep.requireCommitIndex()) {- logger.log("Waiting for commit to be indexed...");- codeIndexManager.indexAsync(jobContext.getProjectId(), jobContext.getCommitId());- while (!codeIndexManager.isIndexed(jobContext.getProjectId(), jobContext.getCommitId())) {- try {- Thread.sleep(1000);- } catch (InterruptedException e) {- throw new RuntimeException(e);- }- }- }-+ var transformedServerSideStep = new EditableStringTransformer(t -> replacePlaceholders(t, placeholderValues)).transformProperties(serverSideStep, Interpolative.class);return transformedServerSideStep.run(jobContext.getBuildId(), inputDir, logger);} else {throw new IllegalStateException("Job actions not found");diff --git a/server-core/src/main/java/io/onedev/server/search/code/CodeSearchManager.java b/server-core/src/main/java/io/onedev/server/search/code/CodeSearchManager.javaindex 8bcced42b1..ff9ea500a4 100644--- a/server-core/src/main/java/io/onedev/server/search/code/CodeSearchManager.java+++ b/server-core/src/main/java/io/onedev/server/search/code/CodeSearchManager.java@@ -1,16 +1,18 @@package io.onedev.server.search.code;+import java.util.List;++import javax.annotation.Nullable;++import org.apache.lucene.search.IndexSearcher;+import org.eclipse.jgit.lib.ObjectId;+import io.onedev.commons.jsymbol.Symbol;import io.onedev.server.model.Project;import io.onedev.server.search.code.hit.QueryHit;import io.onedev.server.search.code.hit.SymbolHit;import io.onedev.server.search.code.query.BlobQuery;import io.onedev.server.search.code.query.TooGeneralQueryException;-import org.apache.lucene.search.IndexSearcher;-import org.eclipse.jgit.lib.ObjectId;--import javax.annotation.Nullable;-import java.util.List;public interface CodeSearchManager {@@ -32,9 +34,6 @@ public interface CodeSearchManager {@NullableList<Symbol> getSymbols(IndexSearcher searcher, ObjectId blobId, String blobPath);- @Nullable- String findBlobPathBySuffix(Project project, ObjectId commit, String blobPathSuffix);-@NullableSymbolHit findPrimarySymbol(Project project, ObjectId commitId, String symbolFQN,String fqnSeparator);diff --git a/server-core/src/main/java/io/onedev/server/search/code/DefaultCodeSearchManager.java b/server-core/src/main/java/io/onedev/server/search/code/DefaultCodeSearchManager.javaindex b63569ffd6..f9ea577db0 100644--- a/server-core/src/main/java/io/onedev/server/search/code/DefaultCodeSearchManager.java+++ b/server-core/src/main/java/io/onedev/server/search/code/DefaultCodeSearchManager.java@@ -1,31 +1,38 @@package io.onedev.server.search.code;-import com.google.common.base.Preconditions;-import io.onedev.commons.jsymbol.Symbol;-import io.onedev.commons.jsymbol.SymbolExtractorRegistry;-import io.onedev.commons.loader.ManagedSerializedForm;-import io.onedev.commons.utils.ExceptionUtils;-import io.onedev.server.cluster.ClusterManager;-import io.onedev.server.cluster.ClusterTask;-import io.onedev.server.entitymanager.ProjectManager;-import io.onedev.server.event.Listen;-import io.onedev.server.event.entity.EntityRemoved;-import io.onedev.server.event.project.CommitIndexed;-import io.onedev.server.event.system.SystemStopping;-import io.onedev.server.model.Project;-import io.onedev.server.persistence.annotation.Transactional;-import io.onedev.server.search.code.hit.QueryHit;-import io.onedev.server.search.code.hit.SymbolHit;-import io.onedev.server.search.code.query.BlobQuery;-import io.onedev.server.search.code.query.FileQuery;-import io.onedev.server.search.code.query.SymbolQuery;+import static io.onedev.server.search.code.FieldConstants.BLOB_HASH;+import static io.onedev.server.search.code.FieldConstants.BLOB_INDEX_VERSION;+import static io.onedev.server.search.code.FieldConstants.BLOB_PATH;+import static io.onedev.server.search.code.FieldConstants.BLOB_SYMBOL_LIST;++import java.io.IOException;+import java.io.ObjectStreamException;+import java.io.Serializable;+import java.nio.channels.ClosedByInterruptException;+import java.util.ArrayList;+import java.util.HashSet;+import java.util.List;+import java.util.Map;+import java.util.Set;+import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.atomic.AtomicReference;++import javax.annotation.Nullable;+import javax.inject.Inject;+import javax.inject.Singleton;+import org.apache.commons.lang3.SerializationUtils;import org.apache.lucene.document.Document;import org.apache.lucene.index.BinaryDocValues;import org.apache.lucene.index.DirectoryReader;import org.apache.lucene.index.LeafReaderContext;import org.apache.lucene.search.BooleanClause.Occur;-import org.apache.lucene.search.*;+import org.apache.lucene.search.BooleanQuery;+import org.apache.lucene.search.CollectionTerminatedException;+import org.apache.lucene.search.IndexSearcher;+import org.apache.lucene.search.ScoreMode;+import org.apache.lucene.search.SearcherManager;+import org.apache.lucene.search.SimpleCollector;import org.apache.lucene.store.Directory;import org.apache.lucene.store.FSDirectory;import org.apache.lucene.util.BytesRef;@@ -37,18 +44,25 @@ import org.eclipse.jgit.treewalk.TreeWalk;import org.slf4j.Logger;import org.slf4j.LoggerFactory;-import javax.annotation.Nullable;-import javax.inject.Inject;-import javax.inject.Singleton;-import java.io.IOException;-import java.io.ObjectStreamException;-import java.io.Serializable;-import java.nio.channels.ClosedByInterruptException;-import java.util.*;-import java.util.concurrent.ConcurrentHashMap;-import java.util.concurrent.atomic.AtomicReference;+import com.google.common.base.Preconditions;-import static io.onedev.server.search.code.FieldConstants.*;+import io.onedev.commons.jsymbol.Symbol;+import io.onedev.commons.jsymbol.SymbolExtractorRegistry;+import io.onedev.commons.loader.ManagedSerializedForm;+import io.onedev.commons.utils.ExceptionUtils;+import io.onedev.server.cluster.ClusterManager;+import io.onedev.server.cluster.ClusterTask;+import io.onedev.server.entitymanager.ProjectManager;+import io.onedev.server.event.Listen;+import io.onedev.server.event.entity.EntityRemoved;+import io.onedev.server.event.project.CommitIndexed;+import io.onedev.server.event.system.SystemStopping;+import io.onedev.server.model.Project;+import io.onedev.server.persistence.annotation.Transactional;+import io.onedev.server.search.code.hit.QueryHit;+import io.onedev.server.search.code.hit.SymbolHit;+import io.onedev.server.search.code.query.BlobQuery;+import io.onedev.server.search.code.query.SymbolQuery;@Singletonpublic class DefaultCodeSearchManager implements CodeSearchManager, Serializable {@@ -318,29 +332,6 @@ public class DefaultCodeSearchManager implements CodeSearchManager, Serializable}}- @Nullable- @Override- public String findBlobPathBySuffix(Project project, ObjectId commit, String blobPathSuffix) {- var fileName = blobPathSuffix;- var lastIndex = fileName.lastIndexOf('/');- if (lastIndex != -1)- fileName = fileName.substring(lastIndex + 1);-- var query = new FileQuery.Builder(fileName).caseSensitive(true).count(MAX_BLOB_PATH_QUERY_COUNT).build();- String blobPath = null;- for (QueryHit hit: search(project, commit, query)) {- if (hit.getBlobPath().endsWith(blobPathSuffix)) {- if (blobPath == null) {- blobPath = hit.getBlobPath();- } else {- blobPath = null;- break;- }- }- }- return blobPath;- }-@Nullable@Overridepublic SymbolHit findPrimarySymbol(Project project, ObjectId commitId, String symbolFQN, String fqnSeparator) {diff --git a/server-plugin/server-plugin-report-jacoco/src/main/java/io/onedev/server/plugin/report/jacoco/PublishJacocoReportStep.java b/server-plugin/server-plugin-report-jacoco/src/main/java/io/onedev/server/plugin/report/jacoco/PublishJacocoReportStep.javaindex 40b62226a3..fd63df048b 100644--- a/server-plugin/server-plugin-report-jacoco/src/main/java/io/onedev/server/plugin/report/jacoco/PublishJacocoReportStep.java+++ b/server-plugin/server-plugin-report-jacoco/src/main/java/io/onedev/server/plugin/report/jacoco/PublishJacocoReportStep.java@@ -1,5 +1,21 @@package io.onedev.server.plugin.report.jacoco;+import java.io.File;+import java.io.IOException;+import java.io.StringReader;+import java.nio.charset.StandardCharsets;+import java.util.ArrayList;+import java.util.HashMap;+import java.util.List;+import java.util.Map;++import javax.validation.constraints.NotEmpty;++import org.dom4j.Document;+import org.dom4j.DocumentException;+import org.dom4j.Element;+import org.dom4j.io.SAXReader;+import io.onedev.commons.codeassist.InputSuggestion;import io.onedev.commons.utils.FileUtils;import io.onedev.commons.utils.TaskLogger;@@ -10,24 +26,16 @@ import io.onedev.server.annotation.Patterns;import io.onedev.server.buildspec.BuildSpec;import io.onedev.server.buildspec.step.StepGroup;import io.onedev.server.codequality.CoverageStatus;+import io.onedev.server.entitymanager.ProjectManager;+import io.onedev.server.git.GitUtils;import io.onedev.server.model.Build;-import io.onedev.server.plugin.report.coverage.*;-import io.onedev.server.search.code.CodeSearchManager;+import io.onedev.server.plugin.report.coverage.Coverage;+import io.onedev.server.plugin.report.coverage.CoverageReport;+import io.onedev.server.plugin.report.coverage.CoverageStats;+import io.onedev.server.plugin.report.coverage.FileCoverage;+import io.onedev.server.plugin.report.coverage.GroupCoverage;+import io.onedev.server.plugin.report.coverage.PublishCoverageReportStep;import io.onedev.server.util.XmlUtils;-import org.dom4j.Document;-import org.dom4j.DocumentException;-import org.dom4j.Element;-import org.dom4j.io.SAXReader;--import javax.validation.constraints.NotEmpty;-import java.io.File;-import java.io.IOException;-import java.io.StringReader;-import java.nio.charset.StandardCharsets;-import java.util.ArrayList;-import java.util.HashMap;-import java.util.List;-import java.util.Map;@Editable(order=10000, group=StepGroup.PUBLISH, name="JaCoCo Coverage Report")public class PublishJacocoReportStep extends PublishCoverageReportStep {@@ -54,11 +62,6 @@ public class PublishJacocoReportStep extends PublishCoverageReportStep {return BuildSpec.suggestVariables(matchWith, true, true, false);}- @Override- public boolean requireCommitIndex() {- return true;- }-@Overrideprotected CoverageReport process(Build build, File inputDir, TaskLogger logger) {int baseLen = inputDir.getAbsolutePath().length() + 1;@@ -70,7 +73,9 @@ public class PublishJacocoReportStep extends PublishCoverageReportStep {Map<String, Map<Integer, CoverageStatus>> coverageStatuses = new HashMap<>();- CodeSearchManager searchManager = OneDev.getInstance(CodeSearchManager.class);+ var projectManager = OneDev.getInstance(ProjectManager.class);+ var repository = projectManager.getRepository(build.getProject().getId());+ var blobPaths = GitUtils.getBlobPaths(repository, build.getCommitId());for (File file: getPatternSet().listFiles(inputDir)) {String relativePath = file.getAbsolutePath().substring(baseLen);logger.log("Processing JaCoCo report '" + relativePath + "'...");@@ -86,8 +91,13 @@ public class PublishJacocoReportStep extends PublishCoverageReportStep {for (Element fileElement: packageElement.elements("sourcefile")) {String fileName = fileElement.attributeValue("name");var fileCoverageInfo = getCoverageInfo(fileElement);- String blobPath = searchManager.findBlobPathBySuffix(build.getProject(), build.getCommitId(),- packageName + "/" + fileName);+ String blobPath = null;+ for (var eachBlobPath: blobPaths) {+ if (eachBlobPath.endsWith(packageName + "/" + fileName)) {+ blobPath = eachBlobPath;+ break;+ }+ }if (blobPath != null) {fileCoverages.add(new FileCoverage(blobPath, fileCoverageInfo));Map<Integer, CoverageStatus> coverageStatusesOfFile = new HashMap<>();diff --git a/server-plugin/server-plugin-report-trx/src/main/java/io/onedev/server/plugin/report/trx/PublishTRXReportStep.java b/server-plugin/server-plugin-report-trx/src/main/java/io/onedev/server/plugin/report/trx/PublishTRXReportStep.javaindex 91931b2494..d621e0d02d 100644--- a/server-plugin/server-plugin-report-trx/src/main/java/io/onedev/server/plugin/report/trx/PublishTRXReportStep.java+++ b/server-plugin/server-plugin-report-trx/src/main/java/io/onedev/server/plugin/report/trx/PublishTRXReportStep.java@@ -51,11 +51,6 @@ public class PublishTRXReportStep extends PublishUnitTestReportStep {return BuildSpec.suggestVariables(matchWith, true, true, false);}- @Override- public boolean requireCommitIndex() {- return true;- }-@Overrideprotected UnitTestReport process(Build build, File inputDir, TaskLogger logger) {SAXReader reader = new SAXReader();