Skip to content

The Unified and Autonomous Development Platform

chore: Refactoring server side step

Robin Shen27c0204bb067
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/ServerSideStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/ServerSideStep.java
index dd881108a7..d277fceef7 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/step/ServerSideStep.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/ServerSideStep.java
@@ -36,11 +36,7 @@ public abstract class ServerSideStep extends Step {
protected PatternSet getFiles() {
return new PatternSet(new HashSet<>(), new HashSet<>());
}
-
- public boolean requireCommitIndex() {
- return false;
- }
-
+
public abstract ServerStepResult run(Long buildId, File inputDir, TaskLogger logger);
@Override
diff --git a/server-core/src/main/java/io/onedev/server/git/GitUtils.java b/server-core/src/main/java/io/onedev/server/git/GitUtils.java
index 5d546e94ae..bf61493805 100644
--- a/server-core/src/main/java/io/onedev/server/git/GitUtils.java
+++ b/server-core/src/main/java/io/onedev/server/git/GitUtils.java
@@ -196,6 +196,24 @@ public class GitUtils {
throw new RuntimeException(e);
}
}
+
+ public static Collection<String> getBlobPaths(Repository repository, ObjectId commitId) {
+ Collection<String> blobPaths = new HashSet<>();
+ try (RevWalk revWalk = new RevWalk(repository);
+ TreeWalk treeWalk = new TreeWalk(repository)) {
+ RevCommit commit = revWalk.parseCommit(commitId);
+ treeWalk.addTree(commit.getTree());
+ treeWalk.setRecursive(true);
+ treeWalk.setFilter(TreeFilter.ANY_DIFF);
+
+ while (treeWalk.next()) {
+ blobPaths.add(treeWalk.getPathString());
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return blobPaths;
+ }
@Nullable
public static RevCommit getLastCommit(Repository repository) {
diff --git a/server-core/src/main/java/io/onedev/server/job/DefaultJobManager.java b/server-core/src/main/java/io/onedev/server/job/DefaultJobManager.java
index e030e8c316..a72b90e4ba 100644
--- a/server-core/src/main/java/io/onedev/server/job/DefaultJobManager.java
+++ b/server-core/src/main/java/io/onedev/server/job/DefaultJobManager.java
@@ -149,7 +149,6 @@ import io.onedev.server.persistence.TransactionManager;
import io.onedev.server.persistence.annotation.Sessional;
import io.onedev.server.persistence.annotation.Transactional;
import io.onedev.server.persistence.dao.Dao;
-import io.onedev.server.search.code.CodeIndexManager;
import io.onedev.server.security.CodePullAuthorizationSource;
import io.onedev.server.security.SecurityUtils;
import io.onedev.server.security.permission.AccessBuild;
@@ -224,8 +223,6 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthoriz
private final Validator validator;
private final ClusterManager clusterManager;
-
- private final CodeIndexManager codeIndexManager;
private final GitService gitService;
@@ -252,7 +249,7 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthoriz
ListenerRegistry listenerRegistry, SettingManager settingManager, TransactionManager transactionManager,
LogManager logManager, ExecutorService executorService, SessionManager sessionManager,
BuildParamManager buildParamManager, ProjectManager projectManager, Validator validator,
- TaskScheduler taskScheduler, ClusterManager clusterManager, CodeIndexManager codeIndexManager,
+ TaskScheduler taskScheduler, ClusterManager clusterManager,
PullRequestManager pullRequestManager, IssueManager issueManager, GitService gitService,
SSLFactory sslFactory, Dao dao, BatchWorkManager batchWorkManager) {
this.dao = dao;
@@ -269,7 +266,6 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthoriz
this.projectManager = projectManager;
this.validator = validator;
this.taskScheduler = taskScheduler;
- this.codeIndexManager = codeIndexManager;
this.clusterManager = clusterManager;
this.pullRequestManager = pullRequestManager;
this.issueManager = issueManager;
@@ -1551,20 +1547,7 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthoriz
if (actions != null) {
ServerSideFacade serverSideFacade = (ServerSideFacade) LeafFacade.of(actions, stepPosition);
var serverSideStep = (ServerSideStep) serverSideFacade.getStep();
- var transformedServerSideStep = new EditableStringTransformer(t -> replacePlaceholders(t, placeholderValues)).transformProperties(serverSideStep, Interpolative.class);
-
- if (transformedServerSideStep.requireCommitIndex()) {
- logger.log("Waiting for commit to be indexed...");
- codeIndexManager.indexAsync(jobContext.getProjectId(), jobContext.getCommitId());
- while (!codeIndexManager.isIndexed(jobContext.getProjectId(), jobContext.getCommitId())) {
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
+ var transformedServerSideStep = new EditableStringTransformer(t -> replacePlaceholders(t, placeholderValues)).transformProperties(serverSideStep, Interpolative.class);
return transformedServerSideStep.run(jobContext.getBuildId(), inputDir, logger);
} else {
throw new IllegalStateException("Job actions not found");
diff --git a/server-core/src/main/java/io/onedev/server/search/code/CodeSearchManager.java b/server-core/src/main/java/io/onedev/server/search/code/CodeSearchManager.java
index 8bcced42b1..ff9ea500a4 100644
--- a/server-core/src/main/java/io/onedev/server/search/code/CodeSearchManager.java
+++ b/server-core/src/main/java/io/onedev/server/search/code/CodeSearchManager.java
@@ -1,16 +1,18 @@
package io.onedev.server.search.code;
+import java.util.List;
+
+import javax.annotation.Nullable;
+
+import org.apache.lucene.search.IndexSearcher;
+import org.eclipse.jgit.lib.ObjectId;
+
import io.onedev.commons.jsymbol.Symbol;
import io.onedev.server.model.Project;
import io.onedev.server.search.code.hit.QueryHit;
import io.onedev.server.search.code.hit.SymbolHit;
import io.onedev.server.search.code.query.BlobQuery;
import io.onedev.server.search.code.query.TooGeneralQueryException;
-import org.apache.lucene.search.IndexSearcher;
-import org.eclipse.jgit.lib.ObjectId;
-
-import javax.annotation.Nullable;
-import java.util.List;
public interface CodeSearchManager {
@@ -32,9 +34,6 @@ public interface CodeSearchManager {
@Nullable
List<Symbol> getSymbols(IndexSearcher searcher, ObjectId blobId, String blobPath);
- @Nullable
- String findBlobPathBySuffix(Project project, ObjectId commit, String blobPathSuffix);
-
@Nullable
SymbolHit findPrimarySymbol(Project project, ObjectId commitId, String symbolFQN,
String fqnSeparator);
diff --git a/server-core/src/main/java/io/onedev/server/search/code/DefaultCodeSearchManager.java b/server-core/src/main/java/io/onedev/server/search/code/DefaultCodeSearchManager.java
index b63569ffd6..f9ea577db0 100644
--- a/server-core/src/main/java/io/onedev/server/search/code/DefaultCodeSearchManager.java
+++ b/server-core/src/main/java/io/onedev/server/search/code/DefaultCodeSearchManager.java
@@ -1,31 +1,38 @@
package io.onedev.server.search.code;
-import com.google.common.base.Preconditions;
-import io.onedev.commons.jsymbol.Symbol;
-import io.onedev.commons.jsymbol.SymbolExtractorRegistry;
-import io.onedev.commons.loader.ManagedSerializedForm;
-import io.onedev.commons.utils.ExceptionUtils;
-import io.onedev.server.cluster.ClusterManager;
-import io.onedev.server.cluster.ClusterTask;
-import io.onedev.server.entitymanager.ProjectManager;
-import io.onedev.server.event.Listen;
-import io.onedev.server.event.entity.EntityRemoved;
-import io.onedev.server.event.project.CommitIndexed;
-import io.onedev.server.event.system.SystemStopping;
-import io.onedev.server.model.Project;
-import io.onedev.server.persistence.annotation.Transactional;
-import io.onedev.server.search.code.hit.QueryHit;
-import io.onedev.server.search.code.hit.SymbolHit;
-import io.onedev.server.search.code.query.BlobQuery;
-import io.onedev.server.search.code.query.FileQuery;
-import io.onedev.server.search.code.query.SymbolQuery;
+import static io.onedev.server.search.code.FieldConstants.BLOB_HASH;
+import static io.onedev.server.search.code.FieldConstants.BLOB_INDEX_VERSION;
+import static io.onedev.server.search.code.FieldConstants.BLOB_PATH;
+import static io.onedev.server.search.code.FieldConstants.BLOB_SYMBOL_LIST;
+
+import java.io.IOException;
+import java.io.ObjectStreamException;
+import java.io.Serializable;
+import java.nio.channels.ClosedByInterruptException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
import org.apache.commons.lang3.SerializationUtils;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.*;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.CollectionTerminatedException;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.SearcherManager;
+import org.apache.lucene.search.SimpleCollector;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.BytesRef;
@@ -37,18 +44,25 @@ import org.eclipse.jgit.treewalk.TreeWalk;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.annotation.Nullable;
-import javax.inject.Inject;
-import javax.inject.Singleton;
-import java.io.IOException;
-import java.io.ObjectStreamException;
-import java.io.Serializable;
-import java.nio.channels.ClosedByInterruptException;
-import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicReference;
+import com.google.common.base.Preconditions;
-import static io.onedev.server.search.code.FieldConstants.*;
+import io.onedev.commons.jsymbol.Symbol;
+import io.onedev.commons.jsymbol.SymbolExtractorRegistry;
+import io.onedev.commons.loader.ManagedSerializedForm;
+import io.onedev.commons.utils.ExceptionUtils;
+import io.onedev.server.cluster.ClusterManager;
+import io.onedev.server.cluster.ClusterTask;
+import io.onedev.server.entitymanager.ProjectManager;
+import io.onedev.server.event.Listen;
+import io.onedev.server.event.entity.EntityRemoved;
+import io.onedev.server.event.project.CommitIndexed;
+import io.onedev.server.event.system.SystemStopping;
+import io.onedev.server.model.Project;
+import io.onedev.server.persistence.annotation.Transactional;
+import io.onedev.server.search.code.hit.QueryHit;
+import io.onedev.server.search.code.hit.SymbolHit;
+import io.onedev.server.search.code.query.BlobQuery;
+import io.onedev.server.search.code.query.SymbolQuery;
@Singleton
public class DefaultCodeSearchManager implements CodeSearchManager, Serializable {
@@ -318,29 +332,6 @@ public class DefaultCodeSearchManager implements CodeSearchManager, Serializable
}
}
- @Nullable
- @Override
- public String findBlobPathBySuffix(Project project, ObjectId commit, String blobPathSuffix) {
- var fileName = blobPathSuffix;
- var lastIndex = fileName.lastIndexOf('/');
- if (lastIndex != -1)
- fileName = fileName.substring(lastIndex + 1);
-
- var query = new FileQuery.Builder(fileName).caseSensitive(true).count(MAX_BLOB_PATH_QUERY_COUNT).build();
- String blobPath = null;
- for (QueryHit hit: search(project, commit, query)) {
- if (hit.getBlobPath().endsWith(blobPathSuffix)) {
- if (blobPath == null) {
- blobPath = hit.getBlobPath();
- } else {
- blobPath = null;
- break;
- }
- }
- }
- return blobPath;
- }
-
@Nullable
@Override
public SymbolHit findPrimarySymbol(Project project, ObjectId commitId, String symbolFQN, String fqnSeparator) {
diff --git a/server-plugin/server-plugin-report-jacoco/src/main/java/io/onedev/server/plugin/report/jacoco/PublishJacocoReportStep.java b/server-plugin/server-plugin-report-jacoco/src/main/java/io/onedev/server/plugin/report/jacoco/PublishJacocoReportStep.java
index 40b62226a3..fd63df048b 100644
--- a/server-plugin/server-plugin-report-jacoco/src/main/java/io/onedev/server/plugin/report/jacoco/PublishJacocoReportStep.java
+++ b/server-plugin/server-plugin-report-jacoco/src/main/java/io/onedev/server/plugin/report/jacoco/PublishJacocoReportStep.java
@@ -1,5 +1,21 @@
package io.onedev.server.plugin.report.jacoco;
+import java.io.File;
+import java.io.IOException;
+import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.validation.constraints.NotEmpty;
+
+import org.dom4j.Document;
+import org.dom4j.DocumentException;
+import org.dom4j.Element;
+import org.dom4j.io.SAXReader;
+
import io.onedev.commons.codeassist.InputSuggestion;
import io.onedev.commons.utils.FileUtils;
import io.onedev.commons.utils.TaskLogger;
@@ -10,24 +26,16 @@ import io.onedev.server.annotation.Patterns;
import io.onedev.server.buildspec.BuildSpec;
import io.onedev.server.buildspec.step.StepGroup;
import io.onedev.server.codequality.CoverageStatus;
+import io.onedev.server.entitymanager.ProjectManager;
+import io.onedev.server.git.GitUtils;
import io.onedev.server.model.Build;
-import io.onedev.server.plugin.report.coverage.*;
-import io.onedev.server.search.code.CodeSearchManager;
+import io.onedev.server.plugin.report.coverage.Coverage;
+import io.onedev.server.plugin.report.coverage.CoverageReport;
+import io.onedev.server.plugin.report.coverage.CoverageStats;
+import io.onedev.server.plugin.report.coverage.FileCoverage;
+import io.onedev.server.plugin.report.coverage.GroupCoverage;
+import io.onedev.server.plugin.report.coverage.PublishCoverageReportStep;
import io.onedev.server.util.XmlUtils;
-import org.dom4j.Document;
-import org.dom4j.DocumentException;
-import org.dom4j.Element;
-import org.dom4j.io.SAXReader;
-
-import javax.validation.constraints.NotEmpty;
-import java.io.File;
-import java.io.IOException;
-import java.io.StringReader;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
@Editable(order=10000, group=StepGroup.PUBLISH, name="JaCoCo Coverage Report")
public class PublishJacocoReportStep extends PublishCoverageReportStep {
@@ -54,11 +62,6 @@ public class PublishJacocoReportStep extends PublishCoverageReportStep {
return BuildSpec.suggestVariables(matchWith, true, true, false);
}
- @Override
- public boolean requireCommitIndex() {
- return true;
- }
-
@Override
protected CoverageReport process(Build build, File inputDir, TaskLogger logger) {
int baseLen = inputDir.getAbsolutePath().length() + 1;
@@ -70,7 +73,9 @@ public class PublishJacocoReportStep extends PublishCoverageReportStep {
Map<String, Map<Integer, CoverageStatus>> coverageStatuses = new HashMap<>();
- CodeSearchManager searchManager = OneDev.getInstance(CodeSearchManager.class);
+ var projectManager = OneDev.getInstance(ProjectManager.class);
+ var repository = projectManager.getRepository(build.getProject().getId());
+ var blobPaths = GitUtils.getBlobPaths(repository, build.getCommitId());
for (File file: getPatternSet().listFiles(inputDir)) {
String relativePath = file.getAbsolutePath().substring(baseLen);
logger.log("Processing JaCoCo report '" + relativePath + "'...");
@@ -86,8 +91,13 @@ public class PublishJacocoReportStep extends PublishCoverageReportStep {
for (Element fileElement: packageElement.elements("sourcefile")) {
String fileName = fileElement.attributeValue("name");
var fileCoverageInfo = getCoverageInfo(fileElement);
- String blobPath = searchManager.findBlobPathBySuffix(build.getProject(), build.getCommitId(),
- packageName + "/" + fileName);
+ String blobPath = null;
+ for (var eachBlobPath: blobPaths) {
+ if (eachBlobPath.endsWith(packageName + "/" + fileName)) {
+ blobPath = eachBlobPath;
+ break;
+ }
+ }
if (blobPath != null) {
fileCoverages.add(new FileCoverage(blobPath, fileCoverageInfo));
Map<Integer, CoverageStatus> coverageStatusesOfFile = new HashMap<>();
diff --git a/server-plugin/server-plugin-report-trx/src/main/java/io/onedev/server/plugin/report/trx/PublishTRXReportStep.java b/server-plugin/server-plugin-report-trx/src/main/java/io/onedev/server/plugin/report/trx/PublishTRXReportStep.java
index 91931b2494..d621e0d02d 100644
--- a/server-plugin/server-plugin-report-trx/src/main/java/io/onedev/server/plugin/report/trx/PublishTRXReportStep.java
+++ b/server-plugin/server-plugin-report-trx/src/main/java/io/onedev/server/plugin/report/trx/PublishTRXReportStep.java
@@ -51,11 +51,6 @@ public class PublishTRXReportStep extends PublishUnitTestReportStep {
return BuildSpec.suggestVariables(matchWith, true, true, false);
}
- @Override
- public boolean requireCommitIndex() {
- return true;
- }
-
@Override
protected UnitTestReport process(Build build, File inputDir, TaskLogger logger) {
SAXReader reader = new SAXReader();