The Unified and Autonomous Development Platform
Refactor artifacts&reports as server steps
Robin Shen96a2fa4c3ab1
diff --git a/pom.xml b/pom.xmlindex d2ab2fe4e6..c5a787f1d0 100644--- a/pom.xml+++ b/pom.xml@@ -322,7 +322,7 @@<dependency><groupId>commons-io</groupId><artifactId>commons-io</artifactId>- <version>2.5</version>+ <version>2.7</version></dependency><dependency><groupId>commons-codec</groupId>@@ -573,8 +573,8 @@</repository></repositories><properties>- <commons.version>2.0.6</commons.version>- <k8shelper.version>2.0.0</k8shelper.version>+ <commons.version>2.0.7</commons.version>+ <k8shelper.version>2.1.0</k8shelper.version><slf4j.version>1.7.5</slf4j.version><logback.version>1.0.11</logback.version><antlr.version>4.7.2</antlr.version>diff --git a/server-core/src/main/java/io/onedev/server/buildspec/job/DefaultJobManager.java b/server-core/src/main/java/io/onedev/server/buildspec/job/DefaultJobManager.javaindex c6ed11603f..d42817a0f6 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/job/DefaultJobManager.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/job/DefaultJobManager.java@@ -1,6 +1,7 @@package io.onedev.server.buildspec.job;import java.io.File;+import java.io.IOException;import java.util.ArrayList;import java.util.Collection;import java.util.Comparator;@@ -11,7 +12,6 @@ import java.util.Iterator;import java.util.List;import java.util.Map;import java.util.Optional;-import java.util.Set;import java.util.UUID;import java.util.concurrent.Callable;import java.util.concurrent.CancellationException;@@ -53,6 +53,11 @@ import io.onedev.commons.utils.LockUtils;import io.onedev.k8shelper.Action;import io.onedev.k8shelper.CacheInstance;import io.onedev.k8shelper.CloneInfo;+import io.onedev.k8shelper.CompositeExecutable;+import io.onedev.k8shelper.Executable;+import io.onedev.k8shelper.LeafExecutable;+import io.onedev.k8shelper.LeafVisitor;+import io.onedev.k8shelper.ServerExecutable;import io.onedev.server.OneDev;import io.onedev.server.buildspec.BuildSpec;import io.onedev.server.buildspec.Service;@@ -65,6 +70,7 @@ import io.onedev.server.buildspec.job.trigger.ScheduleTrigger;import io.onedev.server.buildspec.param.ParamUtils;import io.onedev.server.buildspec.param.spec.ParamSpec;import io.onedev.server.buildspec.param.spec.SecretParam;+import io.onedev.server.buildspec.step.ServerStep;import io.onedev.server.buildspec.step.Step;import io.onedev.server.entitymanager.BuildManager;import io.onedev.server.entitymanager.BuildParamManager;@@ -370,11 +376,7 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthoriz@Overridepublic void run() {- jobLogger.log("Creating server workspace...");- File serverWorkspace = FileUtils.createTempDir("server-workspace");try {- Set<String> includeFiles = new HashSet<>();- Set<String> excludeFiles = new HashSet<>();AtomicInteger maxRetries = new AtomicInteger(0);AtomicInteger retryDelay = new AtomicInteger(0);List<CacheSpec> caches = new ArrayList<>();@@ -388,29 +390,8 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthorizBuild build = buildManager.load(buildId);Build.push(build);try {- jobLogger.log("Retrieving dependency artifacts...");- for (BuildDependence dependence: build.getDependencies()) {- if (dependence.getArtifacts() != null) {- build.retrieveArtifacts(dependence.getDependency(),- dependence.getArtifacts(), serverWorkspace);- }- }-VariableInterpolator interpolator = new VariableInterpolator(build, build.getParamCombination());- String artifacts = interpolator.interpolate(job.getArtifacts());- if (artifacts != null) {- PatternSet patternSet = PatternSet.parse(artifacts);- includeFiles.addAll(patternSet.getIncludes());- excludeFiles.addAll(patternSet.getExcludes());- }- for (JobReport report: job.getReports()) {- report = interpolator.interpolateProperties(report);- PatternSet patternSet = PatternSet.parse(report.getFilePatterns());- includeFiles.addAll(patternSet.getIncludes());- excludeFiles.addAll(patternSet.getExcludes());- }-for (Step step: job.getSteps()) {step = interpolator.interpolateProperties(step);actions.add(step.getAction(build, build.getParamCombination()));@@ -442,11 +423,10 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthorizBuild.push(build);try {return new JobContext(projectName, buildNumber, projectGitDir,- actions, serverWorkspace, job.isRetrieveSource(),- job.getCloneDepth(), cloneInfo, job.getCpuRequirement(),- job.getMemoryRequirement(), commitId, caches,- new PatternSet(includeFiles, excludeFiles), executor.getCacheTTL(),- retried.get(), services, jobLogger) {+ actions, job.isRetrieveSource(), job.getCloneDepth(),+ cloneInfo, job.getCpuRequirement(), job.getMemoryRequirement(),+ commitId, caches, executor.getCacheTTL(), retried.get(),+ services, jobLogger) {@Overridepublic void notifyJobRunning() {@@ -477,6 +457,71 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthoriz});}++ @Override+ public void runServerStep(List<Integer> stepPosition, File filesDir, SimpleLogger logger) {+ sessionManager.run(new Runnable() {++ @Override+ public void run() {+ Executable entryExecutable = new CompositeExecutable(getActions());++ LeafVisitor<LeafExecutable> visitor = new LeafVisitor<LeafExecutable>() {++ @Override+ public LeafExecutable visit(LeafExecutable executable, List<Integer> position) {+ if (position.equals(stepPosition))+ return executable;+ else+ return null;+ }++ };++ ServerExecutable serverExecutable = (ServerExecutable) entryExecutable.traverse(visitor, new ArrayList<>());+ ServerStep serverStep = (ServerStep) serverExecutable.getStep();+ serverStep.run(buildManager.load(buildId), filesDir, logger);+ }++ });+ }++ @Override+ public void copyDependencies(File targetDir) {+ sessionManager.run(new Runnable() {++ @Override+ public void run() {+ Build build = buildManager.load(buildId);+ for (BuildDependence dependence: build.getDependencies()) {+ if (dependence.getArtifacts() != null) {+ Build dependency = dependence.getDependency();+ LockUtils.read(dependency.getArtifactsLockKey(), new Callable<Void>() {++ @Override+ public Void call() throws Exception {+ File artifactsDir = dependency.getArtifactsDir();+ if (artifactsDir.exists()) {+ PatternSet patternSet = PatternSet.parse(dependence.getArtifacts());+ int baseLen = artifactsDir.getAbsolutePath().length() + 1;+ for (File file: patternSet.listFiles(artifactsDir)) {+ try {+ FileUtils.copyFile(file, new File(targetDir, file.getAbsolutePath().substring(baseLen)));+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ }+ }+ return null;+ }++ });+ }+ }+ }++ });+ }};} finally {@@ -558,36 +603,6 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthoriz} catch (Throwable e) {throw maskSecrets(e, jobSecretsToMask);} finally {- try {- transactionManager.run(new Runnable() {-- @Override- public void run() {- Build build = buildManager.load(buildId);- Build.push(build);- VariableInterpolator interpolator = new VariableInterpolator(build, build.getParamCombination());- try {- String artifacts = interpolator.interpolate(job.getArtifacts());- if (artifacts != null) {- jobLogger.log("Publishing job artifacts...");- build.publishArtifacts(serverWorkspace, artifacts);- }-- jobLogger.log("Processing job reports...");- for (JobReport report: job.getReports()) {- report = interpolator.interpolateProperties(report);- report.process(build, serverWorkspace, jobLogger);- }- } finally {- Build.pop();- }- }-- });- FileUtils.deleteDir(serverWorkspace);- } catch (Throwable e) {- throw maskSecrets(e, jobSecretsToMask);- }jobLogger.log("Job finished");}}@@ -1183,4 +1198,9 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthoriz}}+ @Override+ public void runServerStep(String jobToken, List<Integer> stepPosition, File filesDir, SimpleLogger logger) {+ getJobContext(jobToken, true).runServerStep(stepPosition, filesDir, logger);+ }+}\ No newline at end of filediff --git a/server-core/src/main/java/io/onedev/server/buildspec/job/Job.java b/server-core/src/main/java/io/onedev/server/buildspec/job/Job.javaindex bee5d2f160..3b911f060b 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/job/Job.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/job/Job.java@@ -56,8 +56,6 @@ import io.onedev.server.util.validation.annotation.ClassValidating;import io.onedev.server.web.editable.annotation.ChoiceProvider;import io.onedev.server.web.editable.annotation.Editable;import io.onedev.server.web.editable.annotation.Interpolative;-import io.onedev.server.web.editable.annotation.NameOfEmptyValue;-import io.onedev.server.web.editable.annotation.Patterns;import io.onedev.server.web.editable.annotation.RetryCondition;import io.onedev.server.web.editable.annotation.ShowCondition;import io.onedev.server.web.editable.annotation.SuggestionProvider;@@ -101,10 +99,6 @@ public class Job implements NamedElement, Serializable, Validatable {private List<String> requiredServices = new ArrayList<>();- private String artifacts;-- private List<JobReport> reports = new ArrayList<>();-private List<JobTrigger> triggers = new ArrayList<>();private List<CacheSpec> caches = new ArrayList<>();@@ -267,29 +261,6 @@ public class Job implements NamedElement, Serializable, Validatable {return choices;}- @Editable(order=9115, group="Artifacts & Reports", description="Optionally specify files to publish as job artifacts relative to "- + "repository root. Use * or ? for pattern match")- @Interpolative(variableSuggester="suggestVariables")- @Patterns(path=true)- @NameOfEmptyValue("No artifacts")- public String getArtifacts() {- return artifacts;- }-- public void setArtifacts(String artifacts) {- this.artifacts = artifacts;- }-- @Editable(order=9120, group="Artifacts & Reports", description="Add job reports here")- @Valid- public List<JobReport> getReports() {- return reports;- }-- public void setReports(List<JobReport> reports) {- this.reports = reports;- }-@Editable(order=9400, group="More Settings", description="Specify condition to retry build upon failure")@NotEmpty@RetryConditiondiff --git a/server-core/src/main/java/io/onedev/server/buildspec/job/JobContext.java b/server-core/src/main/java/io/onedev/server/buildspec/job/JobContext.javaindex 23517f58ea..70ac9f5ebb 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/job/JobContext.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/job/JobContext.java@@ -13,7 +13,6 @@ import io.onedev.k8shelper.Action;import io.onedev.k8shelper.CloneInfo;import io.onedev.server.buildspec.Service;import io.onedev.server.util.SimpleLogger;-import io.onedev.server.util.patternset.PatternSet;public abstract class JobContext {@@ -23,8 +22,6 @@ public abstract class JobContext {private final File projectGitDir;- private final File serverWorkspace;-private final List<Action> actions;private final boolean retrieveSource;@@ -41,8 +38,6 @@ public abstract class JobContext {private final Collection<CacheSpec> cacheSpecs;- private final PatternSet collectFiles;-private final List<Service> services;private final int cacheTTL;@@ -55,17 +50,15 @@ public abstract class JobContext {private final Map<String, Integer> cacheCounts = new ConcurrentHashMap<>();- public JobContext(String projectName, Long buildNumber,- File projectGitDir, List<Action> actions, File workspace,- boolean retrieveSource, Integer cloneDepth, CloneInfo cloneInfo,- String cpuRequirement, String memoryRequirement, ObjectId commitId,- Collection<CacheSpec> caches, PatternSet collectFiles, int cacheTTL,+ public JobContext(String projectName, Long buildNumber, File projectGitDir,+ List<Action> actions, boolean retrieveSource, Integer cloneDepth,+ CloneInfo cloneInfo, String cpuRequirement, String memoryRequirement,+ ObjectId commitId, Collection<CacheSpec> caches, int cacheTTL,int retried, List<Service> services, SimpleLogger logger) {this.projectName = projectName;this.buildNumber = buildNumber;this.projectGitDir = projectGitDir;this.actions = actions;- this.serverWorkspace = workspace;this.retrieveSource = retrieveSource;this.cloneDepth = cloneDepth;this.cloneInfo = cloneInfo;@@ -73,7 +66,6 @@ public abstract class JobContext {this.memoryRequirement = memoryRequirement;this.commitId = commitId;this.cacheSpecs = caches;- this.collectFiles = collectFiles;this.cacheTTL = cacheTTL;this.retried = retried;this.services = services;@@ -96,10 +88,6 @@ public abstract class JobContext {return actions;}- public File getServerWorkspace() {- return serverWorkspace;- }-public ObjectId getCommitId() {return commitId;}@@ -128,10 +116,6 @@ public abstract class JobContext {return cacheSpecs;}- public PatternSet getCollectFiles() {- return collectFiles;- }-public SimpleLogger getLogger() {return logger;}@@ -160,4 +144,8 @@ public abstract class JobContext {public abstract void reportJobWorkspace(String jobWorkspace);+ public abstract void runServerStep(List<Integer> stepPosition, File filesDir, SimpleLogger logger);++ public abstract void copyDependencies(File targetDir);+}diff --git a/server-core/src/main/java/io/onedev/server/buildspec/job/JobManager.java b/server-core/src/main/java/io/onedev/server/buildspec/job/JobManager.javaindex 7226d6f9a8..56759cd473 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/job/JobManager.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/job/JobManager.java@@ -1,5 +1,6 @@package io.onedev.server.buildspec.job;+import java.io.File;import java.util.Collection;import java.util.Date;import java.util.List;@@ -10,6 +11,7 @@ import org.eclipse.jgit.lib.ObjectId;import io.onedev.k8shelper.CacheInstance;import io.onedev.server.model.Build;import io.onedev.server.model.Project;+import io.onedev.server.util.SimpleLogger;public interface JobManager {@@ -29,4 +31,6 @@ public interface JobManager {void reportJobCaches(String jobToken, Collection<CacheInstance> cacheInstances);+ void runServerStep(String jobToken, List<Integer> stepPosition, File filesDir, SimpleLogger logger);+}diff --git a/server-core/src/main/java/io/onedev/server/buildspec/job/log/DefaultLogManager.java b/server-core/src/main/java/io/onedev/server/buildspec/job/log/DefaultLogManager.javaindex ee13395cd3..72b001fc70 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/job/log/DefaultLogManager.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/job/log/DefaultLogManager.java@@ -25,7 +25,6 @@ import javax.inject.Inject;import javax.inject.Singleton;import org.antlr.v4.runtime.tree.TerminalNode;-import org.apache.commons.io.IOUtils;import org.apache.commons.lang3.StringUtils;import org.joda.time.DateTime;import org.joda.time.format.DateTimeFormat;@@ -434,7 +433,7 @@ public class DefaultLogManager implements LogManager {try {buffer = (renderAsText((JobLogEntry) ois.readObject()) + "\n").getBytes(StandardCharsets.UTF_8);} catch (EOFException e) {- IOUtils.closeQuietly(ois);+ ois.close();ois = null;if (recentBuffer != null) {buffer = recentBuffer;@@ -460,7 +459,7 @@ public class DefaultLogManager implements LogManager {@Overridepublic void close() throws IOException {- IOUtils.closeQuietly(ois);+ ois.close();lock.unlock();}diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/PublishArtifactStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/PublishArtifactStep.javanew file mode 100644index 0000000000..e0d418f03c--- /dev/null+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/PublishArtifactStep.java@@ -0,0 +1,65 @@+package io.onedev.server.buildspec.step;++import java.io.File;+import java.util.List;+import java.util.concurrent.Callable;++import org.hibernate.validator.constraints.NotEmpty;++import io.onedev.commons.codeassist.InputSuggestion;+import io.onedev.commons.utils.FileUtils;+import io.onedev.commons.utils.LockUtils;+import io.onedev.server.buildspec.BuildSpec;+import io.onedev.server.model.Build;+import io.onedev.server.util.SimpleLogger;+import io.onedev.server.util.patternset.PatternSet;+import io.onedev.server.web.editable.annotation.Editable;+import io.onedev.server.web.editable.annotation.Interpolative;+import io.onedev.server.web.editable.annotation.Patterns;++@Editable(order=105, name="Publish Artifacts")+public class PublishArtifactStep extends ServerStep {++ private static final long serialVersionUID = 1L;++ private String artifacts;++ @Editable(order=100, description="Specify files to publish as job artifacts relative to "+ + "repository root. Use * or ? for pattern match")+ @Interpolative(variableSuggester="suggestVariables")+ @Patterns(path=true)+ @NotEmpty+ public String getArtifacts() {+ return artifacts;+ }++ public void setArtifacts(String artifacts) {+ this.artifacts = artifacts;+ }++ @SuppressWarnings("unused")+ private static List<InputSuggestion> suggestVariables(String matchWith) {+ return BuildSpec.suggestVariables(matchWith);+ }++ @Override+ protected PatternSet getFiles() {+ return PatternSet.parse(getArtifacts());+ }++ @Override+ public void run(Build build, File filesDir, SimpleLogger jobLogger) {+ LockUtils.write(build.getArtifactsLockKey(), new Callable<Void>() {++ @Override+ public Void call() throws Exception {+ File artifactsDir = build.getArtifactsDir();+ FileUtils.createDir(artifactsDir);+ FileUtils.copyDirectory(filesDir, artifactsDir);+ return null;+ }++ });+ }++}diff --git a/server-core/src/main/java/io/onedev/server/buildspec/job/JobReport.java b/server-core/src/main/java/io/onedev/server/buildspec/step/PublishReportStep.javasimilarity index 80%rename from server-core/src/main/java/io/onedev/server/buildspec/job/JobReport.javarename to server-core/src/main/java/io/onedev/server/buildspec/step/PublishReportStep.javaindex 35d12743d9..b03ce7d8b6 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/job/JobReport.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/PublishReportStep.java@@ -1,15 +1,11 @@-package io.onedev.server.buildspec.job;+package io.onedev.server.buildspec.step;-import java.io.File;-import java.io.Serializable;import java.util.List;import org.hibernate.validator.constraints.NotEmpty;import io.onedev.commons.codeassist.InputSuggestion;import io.onedev.server.buildspec.BuildSpec;-import io.onedev.server.model.Build;-import io.onedev.server.util.SimpleLogger;import io.onedev.server.util.patternset.PatternSet;import io.onedev.server.util.validation.annotation.PathSegment;import io.onedev.server.web.editable.annotation.Editable;@@ -17,7 +13,7 @@ import io.onedev.server.web.editable.annotation.Interpolative;import io.onedev.server.web.editable.annotation.Patterns;@Editable-public abstract class JobReport implements Serializable {+public abstract class PublishReportStep extends ServerStep {private static final long serialVersionUID = 1L;@@ -38,6 +34,11 @@ public abstract class JobReport implements Serializable {this.reportName = reportName;}+ @Override+ protected PatternSet getFiles() {+ return PatternSet.parse(getFilePatterns());+ }+@Editable(order=100, description="Specify files relative to repository root to publish. Use * or ? for pattern match")@Interpolative(variableSuggester="suggestVariables")@Patterns(path=true)@@ -61,6 +62,4 @@ public abstract class JobReport implements Serializable {return patternSet;}- public abstract void process(Build build, File workspace, SimpleLogger logger);-}diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/ServerStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/ServerStep.javanew file mode 100644index 0000000000..b9dd1aa314--- /dev/null+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/ServerStep.java@@ -0,0 +1,29 @@+package io.onedev.server.buildspec.step;++import java.io.File;+import java.util.HashSet;++import io.onedev.k8shelper.Executable;+import io.onedev.k8shelper.ServerExecutable;+import io.onedev.server.buildspec.param.ParamCombination;+import io.onedev.server.model.Build;+import io.onedev.server.util.SimpleLogger;+import io.onedev.server.util.patternset.PatternSet;++public abstract class ServerStep extends Step {++ private static final long serialVersionUID = 1L;++ @Override+ public Executable getExecutable(Build build, ParamCombination paramCombination) {+ PatternSet files = getFiles();+ return new ServerExecutable(this, files.getIncludes(), files.getExcludes());+ }++ protected PatternSet getFiles() {+ return new PatternSet(new HashSet<>(), new HashSet<>());+ }++ public abstract void run(Build build, File filesDir, SimpleLogger logger);++}diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/UseTemplateStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/UseTemplateStep.javaindex b185c347ae..0d16ee2b9a 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/step/UseTemplateStep.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/UseTemplateStep.java@@ -28,7 +28,7 @@ import io.onedev.server.web.editable.annotation.Editable;import io.onedev.server.web.editable.annotation.OmitName;import io.onedev.server.web.editable.annotation.ParamSpecProvider;-@Editable(order=200, name="Use Step Template", description="Step to use specified template")+@Editable(order=10000, name="Use Step Template", description="Step to use specified template")public class UseTemplateStep extends Step {private static final long serialVersionUID = 1L;diff --git a/server-core/src/main/java/io/onedev/server/migration/VersionedXmlDoc.java b/server-core/src/main/java/io/onedev/server/migration/VersionedXmlDoc.javaindex 9a5a0b0805..05f5a0f236 100644--- a/server-core/src/main/java/io/onedev/server/migration/VersionedXmlDoc.java+++ b/server-core/src/main/java/io/onedev/server/migration/VersionedXmlDoc.java@@ -17,7 +17,6 @@ import java.util.Map;import javax.annotation.Nullable;-import org.apache.commons.io.IOUtils;import org.apache.commons.lang3.ObjectUtils;import org.apache.commons.lang3.ObjectUtils.Null;import org.dom4j.Attribute;@@ -387,9 +386,7 @@ public final class VersionedXmlDoc implements Document, Externalizable {}public void writeToFile(File file, boolean pretty) {- OutputStream os = null;- try {- os = new FileOutputStream(file);+ try (OutputStream os = new FileOutputStream(file)){OutputFormat format = new OutputFormat();format.setIndent(pretty);format.setNewlines(pretty);@@ -398,8 +395,6 @@ public final class VersionedXmlDoc implements Document, Externalizable {writer.write(this);} catch (Exception e) {throw ExceptionUtils.unchecked(e);- } finally {- IOUtils.closeQuietly(os);}}diff --git a/server-core/src/main/java/io/onedev/server/model/Build.java b/server-core/src/main/java/io/onedev/server/model/Build.javaindex 541fa606b9..181dbc4656 100644--- a/server-core/src/main/java/io/onedev/server/model/Build.java+++ b/server-core/src/main/java/io/onedev/server/model/Build.java@@ -832,29 +832,6 @@ public class Build extends AbstractEntity implements Referenceable {return streamPreviousNumbersCache.get(limit);}- public void retrieveArtifacts(Build dependency, String artifacts, File workspaceDir) {- LockUtils.read(dependency.getArtifactsLockKey(), new Callable<Void>() {-- @Override- public Void call() throws Exception {- File artifactsDir = dependency.getArtifactsDir();- if (artifactsDir.exists()) {- PatternSet patternSet = PatternSet.parse(artifacts);- int baseLen = artifactsDir.getAbsolutePath().length() + 1;- for (File file: patternSet.listFiles(artifactsDir)) {- try {- FileUtils.copyFile(file, new File(workspaceDir, file.getAbsolutePath().substring(baseLen)));- } catch (IOException e) {- throw new RuntimeException(e);- }- }- }- return null;- }-- });- }-public String getArtifactsLockKey() {return "build-artifacts:" + getId();}diff --git a/server-core/src/main/java/io/onedev/server/rest/BuildResource.java b/server-core/src/main/java/io/onedev/server/rest/BuildResource.javaindex d0093e171d..eefdec67f6 100644--- a/server-core/src/main/java/io/onedev/server/rest/BuildResource.java+++ b/server-core/src/main/java/io/onedev/server/rest/BuildResource.java@@ -31,8 +31,8 @@ import io.onedev.server.security.SecurityUtils;public class BuildResource {private final BuildManager buildManager;-- @Inject++ @Injectpublic BuildResource(BuildManager buildManager) {this.buildManager = buildManager;}@@ -42,6 +42,7 @@ public class BuildResource {public Response query(@QueryParam("job") String jobName, @QueryParam("commit") String commit,@QueryParam("number") Long number, @QueryParam("offset") Integer offset,@QueryParam("count") Integer count, @Context UriInfo uriInfo) {+EntityCriteria<Build> criteria = buildManager.newCriteria();if (jobName != null)criteria.add(Restrictions.eq("jobName", jobName));diff --git a/server-core/src/main/java/io/onedev/server/util/ExceptionUtils.java b/server-core/src/main/java/io/onedev/server/util/ExceptionUtils.javaindex 7bd7e2f5ed..7cdc7b7910 100644--- a/server-core/src/main/java/io/onedev/server/util/ExceptionUtils.java+++ b/server-core/src/main/java/io/onedev/server/util/ExceptionUtils.java@@ -2,6 +2,7 @@ package io.onedev.server.util;import java.io.IOException;+import javax.annotation.Nullable;import javax.servlet.http.HttpServletRequest;import javax.servlet.http.HttpServletResponse;@@ -16,6 +17,7 @@ import org.slf4j.LoggerFactory;import com.google.common.net.HttpHeaders;import io.onedev.server.OneDev;+import io.onedev.server.web.ExpectedExceptionContribution;public class ExceptionUtils extends io.onedev.commons.utils.ExceptionUtils {@@ -48,4 +50,19 @@ public class ExceptionUtils extends io.onedev.commons.utils.ExceptionUtils {String authcHeader = HttpServletRequest.BASIC_AUTH + " realm=\"" + OneDev.NAME + "\"";response.setHeader(HttpHeaders.WWW_AUTHENTICATE, authcHeader);}++ @Nullable+ public static String getExpectedError(Exception exception) {+ for (ExpectedExceptionContribution contribution: OneDev.getExtensions(ExpectedExceptionContribution.class)) {+ for (Class<? extends Exception> expectedExceptionClass: contribution.getExpectedExceptionClasses()) {+ Exception expectedException = ExceptionUtils.find(exception, expectedExceptionClass);+ if (expectedException != null) {+ if (expectedException.getMessage() != null)+ return expectedException.getMessage();+ }+ }+ }+ return null;+ }+}diff --git a/server-core/src/main/java/io/onedev/server/util/patternset/PatternSet.java b/server-core/src/main/java/io/onedev/server/util/patternset/PatternSet.javaindex 43a426996d..b6a735b65f 100644--- a/server-core/src/main/java/io/onedev/server/util/patternset/PatternSet.java+++ b/server-core/src/main/java/io/onedev/server/util/patternset/PatternSet.java@@ -32,6 +32,10 @@ public class PatternSet implements Serializable {private final Set<String> excludes;+ public PatternSet() {+ this(new HashSet<>(), new HashSet<>());+ }+public PatternSet(Set<String> includes, Set<String> excludes) {this.includes = includes;this.excludes = excludes;diff --git a/server-core/src/main/java/io/onedev/server/web/editable/buildspec/job/report/JobReportBean.java b/server-core/src/main/java/io/onedev/server/web/editable/buildspec/job/report/JobReportBean.javadeleted file mode 100644index f2cfb279dc..0000000000--- a/server-core/src/main/java/io/onedev/server/web/editable/buildspec/job/report/JobReportBean.java+++ /dev/null@@ -1,27 +0,0 @@-package io.onedev.server.web.editable.buildspec.job.report;--import java.io.Serializable;--import javax.validation.constraints.NotNull;--import io.onedev.server.buildspec.job.JobReport;-import io.onedev.server.web.editable.annotation.Editable;--@Editable-public class JobReportBean implements Serializable {-- private static final long serialVersionUID = 1L;-- private JobReport report;-- @Editable(name="Type", order=100)- @NotNull(message="may not be empty")- public JobReport getReport() {- return report;- }-- public void setReport(JobReport report) {- this.report = report;- }--}diff --git a/server-core/src/main/java/io/onedev/server/web/editable/buildspec/job/report/JobReportEditPanel.html b/server-core/src/main/java/io/onedev/server/web/editable/buildspec/job/report/JobReportEditPanel.htmldeleted file mode 100644index 17d3d5a1f3..0000000000--- a/server-core/src/main/java/io/onedev/server/web/editable/buildspec/job/report/JobReportEditPanel.html+++ /dev/null@@ -1,15 +0,0 @@-<wicket:panel>- <form wicket:id="form" class="job-report-edit leave-confirm">- <div class="modal-header">- <h5 id="modal-title" class="modal-title">Job Report</h5>- <button wicket:id="close" type="button" class="close"><wicket:svg href="times" class="icon"/></button>- </div>- <div class="modal-body">- <div wicket:id="editor"></div>- </div>- <div class="modal-footer">- <input wicket:id="save" type="submit" class="dirty-aware btn btn-primary" value="Save">- <a wicket:id="cancel" class="btn btn-secondary">Cancel</a>- </div>- </form>-</wicket:panel>\ No newline at end of filediff --git a/server-core/src/main/java/io/onedev/server/web/editable/buildspec/job/report/JobReportEditPanel.java b/server-core/src/main/java/io/onedev/server/web/editable/buildspec/job/report/JobReportEditPanel.javadeleted file mode 100644index 6a57a9fd40..0000000000--- a/server-core/src/main/java/io/onedev/server/web/editable/buildspec/job/report/JobReportEditPanel.java+++ /dev/null@@ -1,113 +0,0 @@-package io.onedev.server.web.editable.buildspec.job.report;--import java.util.List;--import org.apache.wicket.ajax.AjaxRequestTarget;-import org.apache.wicket.ajax.attributes.AjaxRequestAttributes;-import org.apache.wicket.ajax.markup.html.AjaxLink;-import org.apache.wicket.ajax.markup.html.form.AjaxButton;-import org.apache.wicket.markup.html.form.Form;-import org.apache.wicket.markup.html.panel.Panel;-import org.apache.wicket.request.cycle.RequestCycle;--import io.onedev.server.buildspec.BuildSpecAware;-import io.onedev.server.buildspec.job.JobAware;-import io.onedev.server.buildspec.job.JobReport;-import io.onedev.server.web.ajaxlistener.ConfirmLeaveListener;-import io.onedev.server.web.editable.BeanContext;-import io.onedev.server.web.editable.BeanEditor;--@SuppressWarnings("serial")-abstract class JobReportEditPanel extends Panel implements BuildSpecAware, JobAware {-- private final List<JobReport> reports;-- private final int reportIndex;-- public JobReportEditPanel(String id, List<JobReport> reports, int reportIndex) {- super(id);-- this.reports = reports;- this.reportIndex = reportIndex;- }-- @Override- protected void onInitialize() {- super.onInitialize();-- JobReportBean bean = new JobReportBean();- if (reportIndex != -1)- bean.setReport(reports.get(reportIndex));-- Form<?> form = new Form<Void>("form") {-- @Override- protected void onError() {- super.onError();- RequestCycle.get().find(AjaxRequestTarget.class).add(this);- }-- };-- form.add(new AjaxLink<Void>("close") {-- @Override- protected void updateAjaxAttributes(AjaxRequestAttributes attributes) {- super.updateAjaxAttributes(attributes);- attributes.getAjaxCallListeners().add(new ConfirmLeaveListener(JobReportEditPanel.this));- }-- @Override- public void onClick(AjaxRequestTarget target) {- onCancel(target);- }-- });-- BeanEditor editor = BeanContext.edit("editor", bean);- form.add(editor);- form.add(new AjaxButton("save") {-- @Override- protected void onSubmit(AjaxRequestTarget target, Form<?> form) {- super.onSubmit(target, form);-- JobReport report = bean.getReport();- if (editor.isValid()) {- if (reportIndex != -1) {- reports.set(reportIndex, report);- } else {- reports.add(report);- }- onSave(target);- } else {- target.add(form);- }- }-- });-- form.add(new AjaxLink<Void>("cancel") {-- @Override- protected void updateAjaxAttributes(AjaxRequestAttributes attributes) {- super.updateAjaxAttributes(attributes);- attributes.getAjaxCallListeners().add(new ConfirmLeaveListener(JobReportEditPanel.this));- }-- @Override- public void onClick(AjaxRequestTarget target) {- onCancel(target);- }-- });- form.setOutputMarkupId(true);-- add(form);- }-- protected abstract void onSave(AjaxRequestTarget target);-- protected abstract void onCancel(AjaxRequestTarget target);--}diff --git a/server-core/src/main/java/io/onedev/server/web/editable/buildspec/job/report/JobReportListEditPanel.html b/server-core/src/main/java/io/onedev/server/web/editable/buildspec/job/report/JobReportListEditPanel.htmldeleted file mode 100644index 61cbccfb27..0000000000--- a/server-core/src/main/java/io/onedev/server/web/editable/buildspec/job/report/JobReportListEditPanel.html+++ /dev/null@@ -1,12 +0,0 @@-<wicket:panel>- <div class="job-report-list">- <table wicket:id="reports"></table>- <div class="foot">- <a wicket:id="addNew" title="Add new job report" class="btn btn-light btn-hover-primary btn-block"><wicket:svg href="plus" class="icon"></wicket:svg></a>- </div>- </div>- <wicket:fragment wicket:id="actionColumnFrag">- <a wicket:id="edit" class="btn btn-icon btn-xs btn-light btn-hover-primary mr-1" title="Edit"><wicket:svg href="edit" class="icon"></wicket:svg></a>- <a wicket:id="delete" class="btn btn-icon btn-xs btn-light btn-hover-danger" title="Delete"><wicket:svg href="trash" class="icon"></wicket:svg></a>- </wicket:fragment>-</wicket:panel>\ No newline at end of filediff --git a/server-core/src/main/java/io/onedev/server/web/editable/buildspec/job/report/JobReportListEditPanel.java b/server-core/src/main/java/io/onedev/server/web/editable/buildspec/job/report/JobReportListEditPanel.javadeleted file mode 100644index 99c178ddb9..0000000000--- a/server-core/src/main/java/io/onedev/server/web/editable/buildspec/job/report/JobReportListEditPanel.java+++ /dev/null@@ -1,282 +0,0 @@-package io.onedev.server.web.editable.buildspec.job.report;--import java.io.Serializable;-import java.util.ArrayList;-import java.util.Collections;-import java.util.List;--import org.apache.wicket.Component;-import org.apache.wicket.ajax.AjaxRequestTarget;-import org.apache.wicket.ajax.markup.html.AjaxLink;-import org.apache.wicket.event.IEvent;-import org.apache.wicket.extensions.markup.html.repeater.data.grid.ICellPopulator;-import org.apache.wicket.extensions.markup.html.repeater.data.table.AbstractColumn;-import org.apache.wicket.extensions.markup.html.repeater.data.table.DataTable;-import org.apache.wicket.extensions.markup.html.repeater.data.table.HeadersToolbar;-import org.apache.wicket.extensions.markup.html.repeater.data.table.IColumn;-import org.apache.wicket.extensions.markup.html.repeater.data.table.NoRecordsToolbar;-import org.apache.wicket.markup.ComponentTag;-import org.apache.wicket.markup.html.basic.Label;-import org.apache.wicket.markup.html.panel.Fragment;-import org.apache.wicket.markup.repeater.Item;-import org.apache.wicket.markup.repeater.data.IDataProvider;-import org.apache.wicket.markup.repeater.data.ListDataProvider;-import org.apache.wicket.model.IModel;-import org.apache.wicket.model.Model;-import org.apache.wicket.util.convert.ConversionException;--import io.onedev.server.buildspec.BuildSpec;-import io.onedev.server.buildspec.BuildSpecAware;-import io.onedev.server.buildspec.job.Job;-import io.onedev.server.buildspec.job.JobAware;-import io.onedev.server.buildspec.job.JobReport;-import io.onedev.server.buildspec.param.spec.ParamSpec;-import io.onedev.server.web.behavior.NoRecordsBehavior;-import io.onedev.server.web.behavior.sortable.SortBehavior;-import io.onedev.server.web.behavior.sortable.SortPosition;-import io.onedev.server.web.component.modal.ModalLink;-import io.onedev.server.web.component.modal.ModalPanel;-import io.onedev.server.web.component.svg.SpriteImage;-import io.onedev.server.web.editable.EditableUtils;-import io.onedev.server.web.editable.PropertyDescriptor;-import io.onedev.server.web.editable.PropertyEditor;-import io.onedev.server.web.editable.PropertyUpdating;--@SuppressWarnings("serial")-class JobReportListEditPanel extends PropertyEditor<List<Serializable>> {-- private final List<JobReport> reports;-- public JobReportListEditPanel(String id, PropertyDescriptor propertyDescriptor, IModel<List<Serializable>> model) {- super(id, propertyDescriptor, model);-- reports = new ArrayList<>();- for (Serializable each: model.getObject())- reports.add((JobReport) each);- }-- @Override- protected void onInitialize() {- super.onInitialize();- add(new ModalLink("addNew") {-- @Override- protected Component newContent(String id, ModalPanel modal) {- return new JobReportEditPanel(id, reports, -1) {-- @Override- protected void onCancel(AjaxRequestTarget target) {- modal.close();- }-- @Override- protected void onSave(AjaxRequestTarget target) {- markFormDirty(target);- modal.close();- onPropertyUpdating(target);- target.add(JobReportListEditPanel.this);- }-- @Override- public BuildSpec getBuildSpec() {- return JobReportListEditPanel.this.getBuildSpec();- }-- @Override- public Job getJob() {- return JobReportListEditPanel.this.getJob();- }-- @Override- public List<ParamSpec> getParamSpecs() {- return getJob()!=null? getJob().getParamSpecs(): null;- }-- };- }-- });-- List<IColumn<JobReport, Void>> columns = new ArrayList<>();-- columns.add(new AbstractColumn<JobReport, Void>(Model.of("")) {-- @Override- public void populateItem(Item<ICellPopulator<JobReport>> cellItem, String componentId, IModel<JobReport> rowModel) {- cellItem.add(new SpriteImage(componentId, "grip") {-- @Override- protected void onComponentTag(ComponentTag tag) {- super.onComponentTag(tag);- tag.setName("svg");- tag.put("class", "icon drag-indicator");- }-- });- }-- @Override- public String getCssClass() {- return "minimum actions";- }-- });-- columns.add(new AbstractColumn<JobReport, Void>(Model.of("Type")) {-- @Override- public void populateItem(Item<ICellPopulator<JobReport>> cellItem, String componentId, IModel<JobReport> rowModel) {- cellItem.add(new Label(componentId, EditableUtils.getDisplayName(rowModel.getObject().getClass())));- }- });-- columns.add(new AbstractColumn<JobReport, Void>(Model.of("Name")) {-- @Override- public void populateItem(Item<ICellPopulator<JobReport>> cellItem, String componentId, IModel<JobReport> rowModel) {- cellItem.add(new Label(componentId, rowModel.getObject().getReportName()));- }- });-- columns.add(new AbstractColumn<JobReport, Void>(Model.of("File Patterns")) {-- @Override- public void populateItem(Item<ICellPopulator<JobReport>> cellItem, String componentId, IModel<JobReport> rowModel) {- cellItem.add(new Label(componentId, rowModel.getObject().getFilePatterns()));- }- });-- columns.add(new AbstractColumn<JobReport, Void>(Model.of("")) {-- @Override- public void populateItem(Item<ICellPopulator<JobReport>> cellItem, String componentId, IModel<JobReport> rowModel) {- Fragment fragment = new Fragment(componentId, "actionColumnFrag", JobReportListEditPanel.this);- fragment.add(new ModalLink("edit") {-- @Override- protected Component newContent(String id, ModalPanel modal) {- return new JobReportEditPanel(id, reports, cellItem.findParent(Item.class).getIndex()) {-- @Override- protected void onCancel(AjaxRequestTarget target) {- modal.close();- }-- @Override- protected void onSave(AjaxRequestTarget target) {- markFormDirty(target);- modal.close();- onPropertyUpdating(target);- target.add(JobReportListEditPanel.this);- }-- @Override- public BuildSpec getBuildSpec() {- return JobReportListEditPanel.this.getBuildSpec();- }-- @Override- public Job getJob() {- return JobReportListEditPanel.this.getJob();- }-- @Override- public List<ParamSpec> getParamSpecs() {- return getJob()!=null? getJob().getParamSpecs(): null;- }-- };- }-- });- fragment.add(new AjaxLink<Void>("delete") {-- @Override- public void onClick(AjaxRequestTarget target) {- markFormDirty(target);- reports.remove(rowModel.getObject());- onPropertyUpdating(target);- target.add(JobReportListEditPanel.this);- }-- });- cellItem.add(fragment);- }-- @Override- public String getCssClass() {- return "actions minimum";- }-- });-- IDataProvider<JobReport> dataProvider = new ListDataProvider<JobReport>() {-- @Override- protected List<JobReport> getData() {- return reports;- }-- };-- DataTable<JobReport, Void> dataTable;- add(dataTable = new DataTable<JobReport, Void>("reports", columns, dataProvider, Integer.MAX_VALUE));- dataTable.addTopToolbar(new HeadersToolbar<Void>(dataTable, null));- dataTable.addBottomToolbar(new NoRecordsToolbar(dataTable, Model.of("Not defined")));- dataTable.add(new NoRecordsBehavior());-- dataTable.add(new SortBehavior() {-- @Override- protected void onSort(AjaxRequestTarget target, SortPosition from, SortPosition to) {- int fromIndex = from.getItemIndex();- int toIndex = to.getItemIndex();- if (fromIndex < toIndex) {- for (int i=0; i<toIndex-fromIndex; i++)- Collections.swap(reports, fromIndex+i, fromIndex+i+1);- } else {- for (int i=0; i<fromIndex-toIndex; i++)- Collections.swap(reports, fromIndex-i, fromIndex-i-1);- }- onPropertyUpdating(target);- target.add(JobReportListEditPanel.this);- }-- }.sortable("tbody"));- }-- private BuildSpec getBuildSpec() {- BuildSpecAware buildSpecAware = findParent(BuildSpecAware.class);- if (buildSpecAware != null)- return buildSpecAware.getBuildSpec();- else- return null;- }-- private Job getJob() {- JobAware jobAware = findParent(JobAware.class);- if (jobAware != null)- return jobAware.getJob();- else- return null;- }-- @Override- public void onEvent(IEvent<?> event) {- super.onEvent(event);-- if (event.getPayload() instanceof PropertyUpdating) {- event.stop();- onPropertyUpdating(((PropertyUpdating)event.getPayload()).getHandler());- }- }-- @Override- protected List<Serializable> convertInputToValue() throws ConversionException {- List<Serializable> value = new ArrayList<>();- for (JobReport each: reports)- value.add(each);- return value;- }--}diff --git a/server-core/src/main/java/io/onedev/server/web/editable/buildspec/job/report/JobReportListEditSupport.java b/server-core/src/main/java/io/onedev/server/web/editable/buildspec/job/report/JobReportListEditSupport.javadeleted file mode 100644index a59b4d52ad..0000000000--- a/server-core/src/main/java/io/onedev/server/web/editable/buildspec/job/report/JobReportListEditSupport.java+++ /dev/null@@ -1,60 +0,0 @@-package io.onedev.server.web.editable.buildspec.job.report;--import java.io.Serializable;-import java.util.List;--import org.apache.wicket.Component;-import org.apache.wicket.model.IModel;--import io.onedev.server.buildspec.job.JobReport;-import io.onedev.server.util.ReflectionUtils;-import io.onedev.server.web.editable.EditSupport;-import io.onedev.server.web.editable.EmptyValueLabel;-import io.onedev.server.web.editable.PropertyContext;-import io.onedev.server.web.editable.PropertyDescriptor;-import io.onedev.server.web.editable.PropertyEditor;-import io.onedev.server.web.editable.PropertyViewer;--@SuppressWarnings("serial")-public class JobReportListEditSupport implements EditSupport {-- @Override- public PropertyContext<?> getEditContext(PropertyDescriptor descriptor) {- if (List.class.isAssignableFrom(descriptor.getPropertyClass())) {- Class<?> elementClass = ReflectionUtils.getCollectionElementType(descriptor.getPropertyGetter().getGenericReturnType());- if (elementClass == JobReport.class) {- return new PropertyContext<List<Serializable>>(descriptor) {-- @Override- public PropertyViewer renderForView(String componentId, final IModel<List<Serializable>> model) {- return new PropertyViewer(componentId, descriptor) {-- @Override- protected Component newContent(String id, PropertyDescriptor propertyDescriptor) {- if (model.getObject() != null) {- return new JobReportListViewPanel(id, model.getObject());- } else {- return new EmptyValueLabel(id, propertyDescriptor.getPropertyGetter());- }- }-- };- }-- @Override- public PropertyEditor<List<Serializable>> renderForEdit(String componentId, IModel<List<Serializable>> model) {- return new JobReportListEditPanel(componentId, descriptor, model);- }-- };- }- }- return null;- }-- @Override- public int getPriority() {- return 0;- }--}diff --git a/server-core/src/main/java/io/onedev/server/web/editable/buildspec/job/report/JobReportListViewPanel.html b/server-core/src/main/java/io/onedev/server/web/editable/buildspec/job/report/JobReportListViewPanel.htmldeleted file mode 100644index 77faca7dd1..0000000000--- a/server-core/src/main/java/io/onedev/server/web/editable/buildspec/job/report/JobReportListViewPanel.html+++ /dev/null@@ -1,10 +0,0 @@-<wicket:panel>- <div class="job-report-list">- <div class="body">- <table wicket:id="reports"></table>- </div>- </div>- <wicket:fragment wicket:id="columnFrag">- <a wicket:id="link"><span wicket:id="label"></span></a>- </wicket:fragment>-</wicket:panel>\ No newline at end of filediff --git a/server-core/src/main/java/io/onedev/server/web/editable/buildspec/job/report/JobReportListViewPanel.java b/server-core/src/main/java/io/onedev/server/web/editable/buildspec/job/report/JobReportListViewPanel.javadeleted file mode 100644index e3e48b3b43..0000000000--- a/server-core/src/main/java/io/onedev/server/web/editable/buildspec/job/report/JobReportListViewPanel.java+++ /dev/null@@ -1,197 +0,0 @@-package io.onedev.server.web.editable.buildspec.job.report;--import java.io.Serializable;-import java.util.ArrayList;-import java.util.List;--import org.apache.wicket.Component;-import org.apache.wicket.ajax.AjaxRequestTarget;-import org.apache.wicket.ajax.markup.html.AjaxLink;-import org.apache.wicket.behavior.AttributeAppender;-import org.apache.wicket.extensions.markup.html.repeater.data.grid.ICellPopulator;-import org.apache.wicket.extensions.markup.html.repeater.data.table.AbstractColumn;-import org.apache.wicket.extensions.markup.html.repeater.data.table.DataTable;-import org.apache.wicket.extensions.markup.html.repeater.data.table.HeadersToolbar;-import org.apache.wicket.extensions.markup.html.repeater.data.table.IColumn;-import org.apache.wicket.extensions.markup.html.repeater.data.table.NoRecordsToolbar;-import org.apache.wicket.markup.ComponentTag;-import org.apache.wicket.markup.html.basic.Label;-import org.apache.wicket.markup.html.panel.Fragment;-import org.apache.wicket.markup.html.panel.Panel;-import org.apache.wicket.markup.repeater.Item;-import org.apache.wicket.markup.repeater.data.IDataProvider;-import org.apache.wicket.markup.repeater.data.ListDataProvider;-import org.apache.wicket.model.IModel;-import org.apache.wicket.model.Model;--import io.onedev.server.buildspec.job.JobReport;-import io.onedev.server.web.behavior.NoRecordsBehavior;-import io.onedev.server.web.component.offcanvas.OffCanvasCardPanel;-import io.onedev.server.web.component.offcanvas.OffCanvasPanel;-import io.onedev.server.web.component.svg.SpriteImage;-import io.onedev.server.web.editable.BeanContext;-import io.onedev.server.web.editable.EditableUtils;--@SuppressWarnings("serial")-class JobReportListViewPanel extends Panel {-- private final List<JobReport> reports = new ArrayList<>();-- public JobReportListViewPanel(String id, List<Serializable> elements) {- super(id);-- for (Serializable each: elements)- reports.add((JobReport) each);- }-- @Override- protected void onInitialize() {- super.onInitialize();-- List<IColumn<JobReport, Void>> columns = new ArrayList<>();-- columns.add(new AbstractColumn<JobReport, Void>(Model.of("Type")) {-- @Override- public void populateItem(Item<ICellPopulator<JobReport>> cellItem, String componentId, IModel<JobReport> rowModel) {- cellItem.add(new ColumnFragment(componentId, cellItem.findParent(Item.class).getIndex()) {-- @Override- protected Component newLabel(String componentId) {- return new Label(componentId, EditableUtils.getDisplayName(rowModel.getObject().getClass()));- }-- });- }- });-- columns.add(new AbstractColumn<JobReport, Void>(Model.of("Name")) {-- @Override- public void populateItem(Item<ICellPopulator<JobReport>> cellItem, String componentId, IModel<JobReport> rowModel) {- cellItem.add(new ColumnFragment(componentId, cellItem.findParent(Item.class).getIndex()) {-- @Override- protected Component newLabel(String componentId) {- return new Label(componentId, rowModel.getObject().getReportName());- }-- });- }- });-- columns.add(new AbstractColumn<JobReport, Void>(Model.of("File Patterns")) {-- @Override- public void populateItem(Item<ICellPopulator<JobReport>> cellItem, String componentId, IModel<JobReport> rowModel) {- cellItem.add(new ColumnFragment(componentId, cellItem.findParent(Item.class).getIndex()) {-- @Override- protected Component newLabel(String componentId) {- return new Label(componentId, rowModel.getObject().getFilePatterns());- }-- });- }- });-- columns.add(new AbstractColumn<JobReport, Void>(Model.of("")) {-- @Override- public void populateItem(Item<ICellPopulator<JobReport>> cellItem, String componentId, IModel<JobReport> rowModel) {- cellItem.add(new ColumnFragment(componentId, cellItem.findParent(Item.class).getIndex()) {-- @Override- protected Component newLabel(String componentId) {- return new SpriteImage(componentId, "ellipsis") {-- @Override- protected void onComponentTag(ComponentTag tag) {- super.onComponentTag(tag);- tag.setName("svg");- tag.put("class", "icon");- }-- };- }-- });-- }-- @Override- public String getCssClass() {- return "ellipsis text-right";- }-- });-- IDataProvider<JobReport> dataProvider = new ListDataProvider<JobReport>() {-- @Override- protected List<JobReport> getData() {- return reports;- }-- };-- add(new DataTable<JobReport, Void>("reports", columns, dataProvider, Integer.MAX_VALUE) {-- @Override- protected void onInitialize() {- super.onInitialize();- addTopToolbar(new HeadersToolbar<Void>(this, null));- addBottomToolbar(new NoRecordsToolbar(this, Model.of("Not defined")));- add(new NoRecordsBehavior());- }-- });- }-- private abstract class ColumnFragment extends Fragment {-- private final int index;-- public ColumnFragment(String id, int index) {- super(id, "columnFrag", JobReportListViewPanel.this);- this.index = index;- }-- protected abstract Component newLabel(String componentId);-- @Override- protected void onInitialize() {- super.onInitialize();- AjaxLink<Void> link = new AjaxLink<Void>("link") {-- @Override- public void onClick(AjaxRequestTarget target) {- new OffCanvasCardPanel(target, OffCanvasPanel.Placement.RIGHT, null) {-- @Override- protected Component newTitle(String componentId) {- JobReport report = reports.get(index);- return new Label(componentId, "Report (type: " + EditableUtils.getDisplayName(report.getClass()) + ")");- }-- @Override- protected void onInitialize() {- super.onInitialize();- add(AttributeAppender.append("class", "job-report"));- }-- @Override- protected Component newBody(String id) {- return BeanContext.view(id, reports.get(index));- }-- };- }-- };- link.add(newLabel("label"));- add(link);- }-- }-}diff --git a/server-core/src/main/java/io/onedev/server/web/page/simple/error/GeneralErrorPage.java b/server-core/src/main/java/io/onedev/server/web/page/simple/error/GeneralErrorPage.javaindex 87e63289bd..f1069b1055 100644--- a/server-core/src/main/java/io/onedev/server/web/page/simple/error/GeneralErrorPage.java+++ b/server-core/src/main/java/io/onedev/server/web/page/simple/error/GeneralErrorPage.java@@ -10,10 +10,8 @@ import org.apache.wicket.request.mapper.parameter.PageParameters;import com.google.common.base.Throwables;-import io.onedev.commons.utils.ExceptionUtils;-import io.onedev.server.OneDev;import io.onedev.server.security.SecurityUtils;-import io.onedev.server.web.ExpectedExceptionContribution;+import io.onedev.server.util.ExceptionUtils;import io.onedev.server.web.component.MultilineLabel;import io.onedev.server.web.component.link.ViewStateAwarePageLink;import io.onedev.server.web.page.project.ProjectListPage;@@ -31,15 +29,7 @@ public class GeneralErrorPage extends SimplePage {public GeneralErrorPage(Exception exception) {super(new PageParameters());- for (ExpectedExceptionContribution contribution: OneDev.getExtensions(ExpectedExceptionContribution.class)) {- for (Class<? extends Exception> expectedExceptionClass: contribution.getExpectedExceptionClasses()) {- Exception expectedException = ExceptionUtils.find(exception, expectedExceptionClass);- if (expectedException != null)- title = expectedException.getMessage();- if (title != null)- break;- }- }+ String title = ExceptionUtils.getExpectedError(exception);if (title == null) {title = "An unexpected exception occurred";detailMessage = Throwables.getStackTraceAsString(exception);diff --git a/server-core/src/main/java/org/apache/shiro/web/servlet/ShiroHttpServletRequest.java b/server-core/src/main/java/org/apache/shiro/web/servlet/ShiroHttpServletRequest.javanew file mode 100644index 0000000000..e30bd74d63--- /dev/null+++ b/server-core/src/main/java/org/apache/shiro/web/servlet/ShiroHttpServletRequest.java@@ -0,0 +1,256 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one+ * or more contributor license agreements. See the NOTICE file+ * distributed with this work for additional information+ * regarding copyright ownership. The ASF licenses this file+ * to you under the Apache License, Version 2.0 (the+ * "License"); you may not use this file except in compliance+ * with the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing,+ * software distributed under the License is distributed on an+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+ * KIND, either express or implied. See the License for the+ * specific language governing permissions and limitations+ * under the License.+ */+package org.apache.shiro.web.servlet;++import org.apache.shiro.SecurityUtils;+import org.apache.shiro.session.Session;+import org.apache.shiro.subject.Subject;+import org.apache.shiro.subject.support.DisabledSessionException;+import org.apache.shiro.web.util.WebUtils;+import org.apache.wicket.request.cycle.RequestCycle;++import javax.servlet.ServletContext;+import javax.servlet.http.HttpServletRequest;+import javax.servlet.http.HttpServletRequestWrapper;+import javax.servlet.http.HttpSession;+import java.security.Principal;+++/**+ * A {@code ShiroHttpServletRequest} wraps the Servlet container's original {@code ServletRequest} instance, but ensures+ * that all {@link HttpServletRequest} invocations that require Shiro's support ({@link #getRemoteUser getRemoteUser},+ * {@link #getSession getSession}, etc) can be executed first by Shiro as necessary before allowing the underlying+ * Servlet container instance's method to be invoked.+ *+ * @since 0.2+ */+public class ShiroHttpServletRequest extends HttpServletRequestWrapper {++ //TODO - complete JavaDoc++ //The following 7 constants support the Shiro's implementation of the Servlet Specification+ public static final String COOKIE_SESSION_ID_SOURCE = "cookie";+ public static final String URL_SESSION_ID_SOURCE = "url";+ public static final String REFERENCED_SESSION_ID = ShiroHttpServletRequest.class.getName() + "_REQUESTED_SESSION_ID";+ public static final String REFERENCED_SESSION_ID_IS_VALID = ShiroHttpServletRequest.class.getName() + "_REQUESTED_SESSION_ID_VALID";+ public static final String REFERENCED_SESSION_IS_NEW = ShiroHttpServletRequest.class.getName() + "_REFERENCED_SESSION_IS_NEW";+ public static final String REFERENCED_SESSION_ID_SOURCE = ShiroHttpServletRequest.class.getName() + "REFERENCED_SESSION_ID_SOURCE";+ public static final String IDENTITY_REMOVED_KEY = ShiroHttpServletRequest.class.getName() + "_IDENTITY_REMOVED_KEY";+ public static final String SESSION_ID_URL_REWRITING_ENABLED = ShiroHttpServletRequest.class.getName() + "_SESSION_ID_URL_REWRITING_ENABLED";++ protected ServletContext servletContext = null;++ protected HttpSession session = null;+ protected boolean httpSessions = true;++ public ShiroHttpServletRequest(HttpServletRequest wrapped, ServletContext servletContext, boolean httpSessions) {+ super(wrapped);+ this.servletContext = servletContext;+ this.httpSessions = httpSessions;+ }++ public boolean isHttpSessions() {+ return httpSessions;+ }++ public String getRemoteUser() {+ String remoteUser;+ Object scPrincipal = getSubjectPrincipal();+ if (scPrincipal != null) {+ if (scPrincipal instanceof String) {+ return (String) scPrincipal;+ } else if (scPrincipal instanceof Principal) {+ remoteUser = ((Principal) scPrincipal).getName();+ } else {+ remoteUser = scPrincipal.toString();+ }+ } else {+ remoteUser = super.getRemoteUser();+ }+ return remoteUser;+ }++ protected Subject getSubject() {+ return SecurityUtils.getSubject();+ }++ protected Object getSubjectPrincipal() {+ Object userPrincipal = null;+ Subject subject = getSubject();+ if (subject != null) {+ userPrincipal = subject.getPrincipal();+ }+ return userPrincipal;+ }++ public boolean isUserInRole(String s) {+ Subject subject = getSubject();+ boolean inRole = (subject != null && subject.hasRole(s));+ if (!inRole) {+ inRole = super.isUserInRole(s);+ }+ return inRole;+ }++ public Principal getUserPrincipal() {+ Principal userPrincipal;+ Object scPrincipal = getSubjectPrincipal();+ if (scPrincipal != null) {+ if (scPrincipal instanceof Principal) {+ userPrincipal = (Principal) scPrincipal;+ } else {+ userPrincipal = new ObjectPrincipal(scPrincipal);+ }+ } else {+ userPrincipal = super.getUserPrincipal();+ }+ return userPrincipal;+ }++ public String getRequestedSessionId() {+ String requestedSessionId = null;+ if (isHttpSessions()) {+ requestedSessionId = super.getRequestedSessionId();+ } else {+ Object sessionId = getAttribute(REFERENCED_SESSION_ID);+ if (sessionId != null) {+ requestedSessionId = sessionId.toString();+ }+ }++ return requestedSessionId;+ }++ public HttpSession getSession(boolean create) {++ HttpSession httpSession;++ if (isHttpSessions()) {+ httpSession = super.getSession(false);+ if (httpSession == null && create) {+ //Shiro 1.2: assert that creation is enabled (SHIRO-266):+ if (RequestCycle.get() != null || WebUtils._isSessionCreationEnabled(this)) {+ httpSession = super.getSession(create);+ } else {+ throw newNoSessionCreationException();+ }+ }+ } else {+ boolean existing = getSubject().getSession(false) != null;++ if (this.session == null || !existing) {+ Session shiroSession = getSubject().getSession(create);+ if (shiroSession != null) {+ this.session = new ShiroHttpSession(shiroSession, this, this.servletContext);+ if (!existing) {+ setAttribute(REFERENCED_SESSION_IS_NEW, Boolean.TRUE);+ }+ } else if (this.session != null) {+ this.session = null;+ }+ }+ httpSession = this.session;+ }++ return httpSession;+ }++ /**+ * Constructs and returns a {@link DisabledSessionException} with an appropriate message explaining why+ * session creation has been disabled.+ *+ * @return a new DisabledSessionException with appropriate no creation message+ * @since 1.2+ */+ private DisabledSessionException newNoSessionCreationException() {+ String msg = "Session creation has been disabled for the current request. This exception indicates " ++ "that there is either a programming error (using a session when it should never be " ++ "used) or that Shiro's configuration needs to be adjusted to allow Sessions to be created " ++ "for the current request. See the " + DisabledSessionException.class.getName() + " JavaDoc " ++ "for more.";+ return new DisabledSessionException(msg);+ }++ public HttpSession getSession() {+ return getSession(true);+ }++ public boolean isRequestedSessionIdValid() {+ if (isHttpSessions()) {+ return super.isRequestedSessionIdValid();+ } else {+ Boolean value = (Boolean) getAttribute(REFERENCED_SESSION_ID_IS_VALID);+ return (value != null && value.equals(Boolean.TRUE));+ }+ }++ public boolean isRequestedSessionIdFromCookie() {+ if (isHttpSessions()) {+ return super.isRequestedSessionIdFromCookie();+ } else {+ String value = (String) getAttribute(REFERENCED_SESSION_ID_SOURCE);+ return value != null && value.equals(COOKIE_SESSION_ID_SOURCE);+ }+ }++ public boolean isRequestedSessionIdFromURL() {+ if (isHttpSessions()) {+ return super.isRequestedSessionIdFromURL();+ } else {+ String value = (String) getAttribute(REFERENCED_SESSION_ID_SOURCE);+ return value != null && value.equals(URL_SESSION_ID_SOURCE);+ }+ }++ public boolean isRequestedSessionIdFromUrl() {+ return isRequestedSessionIdFromURL();+ }++ private class ObjectPrincipal implements java.security.Principal {+ private Object object = null;++ public ObjectPrincipal(Object object) {+ this.object = object;+ }++ public Object getObject() {+ return object;+ }++ public String getName() {+ return getObject().toString();+ }++ public int hashCode() {+ return object.hashCode();+ }++ public boolean equals(Object o) {+ if (o instanceof ObjectPrincipal) {+ ObjectPrincipal op = (ObjectPrincipal) o;+ return getObject().equals(op.getObject());+ }+ return false;+ }++ public String toString() {+ return object.toString();+ }+ }+}diff --git a/server-plugin/pom.xml b/server-plugin/pom.xmlindex 610fb0478b..a55e4b64ef 100644--- a/server-plugin/pom.xml+++ b/server-plugin/pom.xml@@ -29,4 +29,4 @@<module>server-plugin-report-clover</module><module>server-plugin-report-checkstyle</module></modules>-</project>\ No newline at end of file+</project>diff --git a/server-plugin/server-plugin-executor-docker/src/main/java/io/onedev/server/plugin/docker/DockerExecutor.java b/server-plugin/server-plugin-executor-docker/src/main/java/io/onedev/server/plugin/docker/DockerExecutor.javaindex 234cac11ef..867cab6ad8 100644--- a/server-plugin/server-plugin-executor-docker/src/main/java/io/onedev/server/plugin/docker/DockerExecutor.java+++ b/server-plugin/server-plugin-executor-docker/src/main/java/io/onedev/server/plugin/docker/DockerExecutor.java@@ -30,6 +30,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;import com.google.common.base.Joiner;import com.google.common.base.Preconditions;import com.google.common.base.Splitter;+import com.google.common.base.Throwables;import io.onedev.commons.launcher.bootstrap.Bootstrap;import io.onedev.commons.launcher.loader.AppLoader;@@ -43,10 +44,12 @@ import io.onedev.commons.utils.command.LineConsumer;import io.onedev.commons.utils.command.ProcessKiller;import io.onedev.k8shelper.CacheInstance;import io.onedev.k8shelper.CommandExecutable;-import io.onedev.k8shelper.CommandHandler;import io.onedev.k8shelper.CompositeExecutable;import io.onedev.k8shelper.Executable;import io.onedev.k8shelper.KubernetesHelper;+import io.onedev.k8shelper.LeafExecutable;+import io.onedev.k8shelper.LeafHandler;+import io.onedev.k8shelper.ServerExecutable;import io.onedev.k8shelper.SshCloneInfo;import io.onedev.server.OneDev;import io.onedev.server.buildspec.Service;@@ -57,10 +60,12 @@ import io.onedev.server.git.config.GitConfig;import io.onedev.server.model.support.RegistryLogin;import io.onedev.server.model.support.administration.jobexecutor.JobExecutor;import io.onedev.server.plugin.docker.DockerExecutor.TestData;+import io.onedev.server.util.ExceptionUtils;import io.onedev.server.util.PKCS12CertExtractor;import io.onedev.server.util.ServerConfig;import io.onedev.server.util.SimpleLogger;import io.onedev.server.util.concurrent.CapacityRunner;+import io.onedev.server.util.patternset.PatternSet;import io.onedev.server.util.validation.Validatable;import io.onedev.server.util.validation.annotation.ClassValidating;import io.onedev.server.web.editable.annotation.Editable;@@ -409,7 +414,6 @@ public class DockerExecutor extends JobExecutor implements Testable<TestData>, VSimpleLogger jobLogger = jobContext.getLogger();getCapacityRunner().call(new Callable<Void>() {- @SuppressWarnings("resource")@Overridepublic Void call() {String network = getName() + "-" + jobContext.getProjectName() + "-"@@ -563,11 +567,7 @@ public class DockerExecutor extends JobExecutor implements Testable<TestData>, V}jobLogger.log("Copying job dependencies...");- try {- FileUtils.copyDirectory(jobContext.getServerWorkspace(), hostWorkspace);- } catch (IOException e) {- throw new RuntimeException(e);- }+ jobContext.copyDependencies(hostWorkspace);String containerBuildHome;String containerWorkspace;@@ -594,32 +594,33 @@ public class DockerExecutor extends JobExecutor implements Testable<TestData>, VjobContext.reportJobWorkspace(containerWorkspace);Executable entryExecutable = new CompositeExecutable(jobContext.getActions());- try {- List<String> errorMessages = new ArrayList<>();-- entryExecutable.execute(new CommandHandler() {-- @Override- public boolean execute(CommandExecutable executable, List<Integer> position) {++ List<String> errorMessages = new ArrayList<>();++ entryExecutable.execute(new LeafHandler() {++ @Override+ public boolean execute(LeafExecutable executable, List<Integer> position) {+ String positionStr = KubernetesHelper.stringifyPosition(position);+ if (executable instanceof CommandExecutable) {+ CommandExecutable commandExecutable = (CommandExecutable) executable;if (SystemUtils.IS_OS_WINDOWS) {File scriptFile = new File(hostBuildHome, "job-commands.bat");try {- FileUtils.writeLines(scriptFile, executable.getCommands(), "\r\n");+ FileUtils.writeLines(scriptFile, commandExecutable.getCommands(), "\r\n");} catch (IOException e) {throw new RuntimeException(e);}} else {File scriptFile = new File(hostBuildHome, "job-commands.sh");try {- FileUtils.writeLines(scriptFile, executable.getCommands(), "\n");+ FileUtils.writeLines(scriptFile, commandExecutable.getCommands(), "\n");} catch (IOException e) {throw new RuntimeException(e);}}- String stepName = KubernetesHelper.describe(position);-- String containerName = network + "-step-" + stepName;+ String containerName = network + "-step-" + positionStr;Commandline docker = newDocker();docker.clearArgs();docker.addArgs("run", "--name=" + containerName, "--network=" + network);@@ -651,11 +652,11 @@ public class DockerExecutor extends JobExecutor implements Testable<TestData>, V}docker.addArgs("-w", containerWorkspace, "--entrypoint=" + containerEntryPoint);- docker.addArgs(executable.getImage());+ docker.addArgs(commandExecutable.getImage());docker.addArgs(containerCommand);- jobLogger.log("Running step #" + stepName + "...");+ jobLogger.log("Running step #" + positionStr + "...");ExecutionResult result = docker.execute(logger, logger, null, new ProcessKiller() {@@ -683,34 +684,51 @@ public class DockerExecutor extends JobExecutor implements Testable<TestData>, V});if (result.getReturnCode() != 0) {- errorMessages.add("Step #" + stepName + ": Command failed with exit code " + result.getReturnCode());+ errorMessages.add("Step #" + positionStr + ": Command failed with exit code " + result.getReturnCode());return false;} else {return true;}- }-- @Override- public void skip(CommandExecutable executable, List<Integer> position) {- jobLogger.log("Skipping step #" + KubernetesHelper.describe(position) + "...");- }-- }, new ArrayList<>());+ } else {+ jobLogger.log("Running step #" + positionStr + "...");++ ServerExecutable serverExecutable = (ServerExecutable) executable;+ PatternSet filePatternSet = new PatternSet(serverExecutable.getIncludeFiles(),+ serverExecutable.getExcludeFiles());+ File filesDir = FileUtils.createTempDir();+ try {+ int baseLen = hostWorkspace.getAbsolutePath().length()+1;+ for (File file: filePatternSet.listFiles(hostWorkspace)) {+ try {+ FileUtils.copyFile(file, new File(filesDir, file.getAbsolutePath().substring(baseLen)));+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ }- if (!errorMessages.isEmpty())- throw new ExplicitException(errorMessages.iterator().next());- } finally {- jobLogger.log("Sending job outcomes...");-- int baseLen = hostWorkspace.getAbsolutePath().length()+1;- for (File file: jobContext.getCollectFiles().listFiles(hostWorkspace)) {- try {- FileUtils.copyFile(file, new File(jobContext.getServerWorkspace(), file.getAbsolutePath().substring(baseLen)));- } catch (IOException e) {- throw new RuntimeException(e);+ jobContext.runServerStep(position, filesDir, jobLogger);+ return true;+ } catch (Exception e) {+ String errorMessage = ExceptionUtils.getExpectedError(e);+ if (errorMessage == null)+ errorMessage = Throwables.getStackTraceAsString(e);+ errorMessages.add("Step #" + positionStr + " is failed: " + errorMessage);+ return false;+ } finally {+ FileUtils.deleteDir(filesDir);+ }}}- }++ @Override+ public void skip(LeafExecutable executable, List<Integer> position) {+ jobLogger.log("Skipping step #" + KubernetesHelper.stringifyPosition(position) + "...");+ }++ }, new ArrayList<>());++ if (!errorMessages.isEmpty())+ throw new ExplicitException(errorMessages.iterator().next());jobLogger.log("Reporting job caches...");diff --git a/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesExecutor.java b/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesExecutor.javaindex c71c2884ef..e136fa52d0 100644--- a/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesExecutor.java+++ b/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesExecutor.java@@ -3,7 +3,7 @@ package io.onedev.server.plugin.executor.kubernetes;import static io.onedev.k8shelper.KubernetesHelper.ENV_JOB_TOKEN;import static io.onedev.k8shelper.KubernetesHelper.ENV_SERVER_URL;import static io.onedev.k8shelper.KubernetesHelper.LOG_END_MESSAGE;-import static io.onedev.k8shelper.KubernetesHelper.describe;+import static io.onedev.k8shelper.KubernetesHelper.stringifyPosition;import java.io.ByteArrayOutputStream;import java.io.File;@@ -52,9 +52,10 @@ import io.onedev.commons.utils.command.ExecutionResult;import io.onedev.commons.utils.command.LineConsumer;import io.onedev.k8shelper.Action;import io.onedev.k8shelper.CommandExecutable;-import io.onedev.k8shelper.CommandVisitor;+import io.onedev.k8shelper.LeafVisitor;import io.onedev.k8shelper.CompositeExecutable;import io.onedev.k8shelper.ExecuteCondition;+import io.onedev.k8shelper.LeafExecutable;import io.onedev.server.OneDev;import io.onedev.server.buildspec.Service;import io.onedev.server.buildspec.job.CacheSpec;@@ -816,24 +817,48 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestDataList<String> containerNames = Lists.newArrayList("init");- entryExecutable.traverse(new CommandVisitor<Void>() {+ String helperImageVersion;+ try (InputStream is = KubernetesExecutor.class.getClassLoader().getResourceAsStream("k8s-helper-version.properties")) {+ ByteArrayOutputStream baos = new ByteArrayOutputStream();+ IOUtils.copy(is, baos);+ helperImageVersion = baos.toString();+ } catch (IOException e) {+ throw new RuntimeException(e);+ }++ List<Map<Object, Object>> envs = new ArrayList<>();+ envs.add(CollectionUtils.newLinkedHashMap(+ "name", ENV_SERVER_URL,+ "value", getServerUrl()));+ envs.add(CollectionUtils.newLinkedHashMap(+ "name", ENV_JOB_TOKEN,+ "value", jobToken));++ entryExecutable.traverse(new LeafVisitor<Void>() {@Override- public Void visit(CommandExecutable executable, List<Integer> position) {+ public Void visit(LeafExecutable executable, List<Integer> position) {String containerName = getContainerName(position);containerNames.add(containerName);+ String image;+ if (executable instanceof CommandExecutable)+ image = ((CommandExecutable)executable).getImage();+ else+ image = "1dev/k8s-helper-" + baselineOsInfo.getHelperImageSuffix() + ":" + helperImageVersion;+Map<Object, Object> stepContainerSpec = CollectionUtils.newHashMap("name", containerName,- "image", executable.getImage());+ "image", image);if (baselineOsInfo.isLinux()) {stepContainerSpec.put("command", Lists.newArrayList("sh"));- stepContainerSpec.put("args", Lists.newArrayList(containerCommandHome + "/" + describe(position) + ".sh"));+ stepContainerSpec.put("args", Lists.newArrayList(containerCommandHome + "/" + stringifyPosition(position) + ".sh"));} else {stepContainerSpec.put("command", Lists.newArrayList("cmd"));- stepContainerSpec.put("args", Lists.newArrayList("/c", containerCommandHome + "\\" + describe(position) + ".bat"));+ stepContainerSpec.put("args", Lists.newArrayList("/c", containerCommandHome + "\\" + stringifyPosition(position) + ".bat"));}-++ stepContainerSpec.put("env", envs);stepContainerSpec.put("volumeMounts", volumeMounts);containerSpecs.add(stepContainerSpec);@@ -860,23 +885,6 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestDatainitArgs.add("test");}- String helperImageVersion;- try (InputStream is = KubernetesExecutor.class.getClassLoader().getResourceAsStream("k8s-helper-version.properties")) {- ByteArrayOutputStream baos = new ByteArrayOutputStream();- IOUtils.copy(is, baos);- helperImageVersion = baos.toString();- } catch (IOException e) {- throw new RuntimeException(e);- }-- List<Map<Object, Object>> envs = new ArrayList<>();- envs.add(CollectionUtils.newLinkedHashMap(- "name", ENV_SERVER_URL,- "value", getServerUrl()));- envs.add(CollectionUtils.newLinkedHashMap(- "name", ENV_JOB_TOKEN,- "value", jobToken));-Map<Object, Object> initContainerSpec = CollectionUtils.newHashMap("name", "init","image", "1dev/k8s-helper-" + baselineOsInfo.getHelperImageSuffix() + ":" + helperImageVersion,@@ -1067,7 +1075,7 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData}private String getContainerName(List<Integer> stepPosition) {- return "step-" + describe(stepPosition);+ return "step-" + stringifyPosition(stepPosition);}private Map<String, ContainerError> getContainerErrors(Collection<JsonNode> containerStatusNodes) {diff --git a/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesResource.java b/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesResource.javaindex e2a1fb407e..b64dcde658 100644--- a/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesResource.java+++ b/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesResource.java@@ -1,12 +1,15 @@package io.onedev.server.plugin.executor.kubernetes;+import java.io.File;import java.io.IOException;import java.io.InputStream;import java.io.OutputStream;import java.io.Serializable;+import java.nio.ByteBuffer;import java.util.ArrayList;import java.util.Collection;import java.util.HashMap;+import java.util.List;import java.util.Map;import java.util.UUID;@@ -24,12 +27,15 @@ import javax.ws.rs.core.MediaType;import javax.ws.rs.core.Response;import javax.ws.rs.core.StreamingOutput;+import org.apache.commons.compress.utils.IOUtils;import org.apache.commons.lang.SerializationUtils;import com.google.common.base.Splitter;+import com.google.common.base.Throwables;import com.google.common.collect.Lists;import io.onedev.commons.utils.ExplicitException;+import io.onedev.commons.utils.FileUtils;import io.onedev.commons.utils.StringUtils;import io.onedev.commons.utils.TarUtils;import io.onedev.k8shelper.CacheAllocationRequest;@@ -37,6 +43,8 @@ import io.onedev.k8shelper.CacheInstance;import io.onedev.server.buildspec.job.Job;import io.onedev.server.buildspec.job.JobContext;import io.onedev.server.buildspec.job.JobManager;+import io.onedev.server.util.ExceptionUtils;+import io.onedev.server.util.SimpleLogger;@Path("/k8s")@Consumes(MediaType.WILDCARD)@@ -69,8 +77,6 @@ public class KubernetesResource {contextMap.put("projectName", context.getProjectName());contextMap.put("cloneInfo", context.getCloneInfo());contextMap.put("commitHash", context.getCommitId().name());- contextMap.put("collectFiles.includes", context.getCollectFiles().getIncludes());- contextMap.put("collectFiles.excludes", context.getCollectFiles().getExcludes());return SerializationUtils.serialize((Serializable) contextMap);}@@ -94,6 +100,52 @@ public class KubernetesResource {jobManager.reportJobCaches(getJobToken(), cacheInstances);}+ @Path("/run-server-step")+ @Consumes(MediaType.APPLICATION_OCTET_STREAM)+ @POST+ public Response runServerStep(InputStream is) {+ File filesDir = FileUtils.createTempDir();+ try {+ byte[] intBytes = new byte[4];+ if (IOUtils.readFully(is, intBytes) != intBytes.length)+ throw new ExplicitException("Invalid input stream to run server step");+ int length = ByteBuffer.wrap(intBytes).getInt();+ List<Integer> stepPosition = new ArrayList<>();+ for (int i=0; i<length; i++) {+ if (IOUtils.readFully(is, intBytes) != intBytes.length)+ throw new ExplicitException("Invalid input stream to run server step");+ stepPosition.add(ByteBuffer.wrap(intBytes).getInt());+ }++ TarUtils.untar(is, filesDir);++ List<String> logMessages = new ArrayList<>();+ jobManager.runServerStep(getJobToken(), stepPosition, filesDir, new SimpleLogger() {++ @Override+ public void log(String message) {+ logMessages.add(message);+ }++ });++ byte[] logBytes = SerializationUtils.serialize((Serializable) logMessages);++ /*+ * Send log statements back to job pod for logging as otherwise log statements+ * from command step and server step may be dis-ordered+ */+ return Response.ok(logBytes).build();+ } catch (Exception e) {+ String errorMessage = ExceptionUtils.getExpectedError(e);+ if (errorMessage == null)+ errorMessage = Throwables.getStackTraceAsString(e);+ return Response.serverError().entity(errorMessage).build();+ } finally {+ FileUtils.deleteDir(filesDir);+ }+ }+@Path("/download-dependencies")@Produces(MediaType.APPLICATION_OCTET_STREAM)@GET@@ -103,24 +155,20 @@ public class KubernetesResource {@Overridepublic void write(OutputStream output) throws IOException {JobContext context = jobManager.getJobContext(getJobToken(), true);- TarUtils.tar(context.getServerWorkspace(), Lists.newArrayList("**"),- new ArrayList<>(), output);- output.flush();+ File tempDir = FileUtils.createTempDir();+ try {+ context.copyDependencies(tempDir);+ TarUtils.tar(tempDir, Lists.newArrayList("**"), new ArrayList<>(), output);+ output.flush();+ } finally {+ FileUtils.deleteDir(tempDir);+ }}};return Response.ok(os).build();}-- @POST- @Path("/upload-outcomes")- @Consumes(MediaType.APPLICATION_OCTET_STREAM)- public Response uploadOutcomes(InputStream is) {- JobContext context = jobManager.getJobContext(getJobToken(), true);- TarUtils.untar(is, context.getServerWorkspace());- return Response.ok().build();- }-+@GET@Path("/test")public Response test() {diff --git a/server-plugin/server-plugin-report-checkstyle/src/main/java/io/onedev/server/plugin/report/checkstyle/CheckstylePluginModule.java b/server-plugin/server-plugin-report-checkstyle/src/main/java/io/onedev/server/plugin/report/checkstyle/CheckstylePluginModule.javaindex 1bc7a878e6..0f61128797 100644--- a/server-plugin/server-plugin-report-checkstyle/src/main/java/io/onedev/server/plugin/report/checkstyle/CheckstylePluginModule.java+++ b/server-plugin/server-plugin-report-checkstyle/src/main/java/io/onedev/server/plugin/report/checkstyle/CheckstylePluginModule.java@@ -19,7 +19,7 @@ import io.onedev.commons.launcher.loader.AbstractPluginModule;import io.onedev.commons.launcher.loader.ImplementationProvider;import io.onedev.commons.utils.LockUtils;import io.onedev.server.OneDev;-import io.onedev.server.buildspec.job.JobReport;+import io.onedev.server.buildspec.step.PublishReportStep;import io.onedev.server.code.CodeProblem;import io.onedev.server.code.CodeProblemContribution;import io.onedev.server.entitymanager.BuildMetricManager;@@ -53,12 +53,12 @@ public class CheckstylePluginModule extends AbstractPluginModule {@Overridepublic Class<?> getAbstractClass() {- return JobReport.class;+ return PublishReportStep.class;}@Overridepublic Collection<Class<?>> getImplementations() {- return Sets.newHashSet(JobESLintReport.class, JobCheckstyleReport.class);+ return Sets.newHashSet(PublishESLintReportStep.class, PublishCheckstyleReportStep.class);}});@@ -88,18 +88,18 @@ public class CheckstylePluginModule extends AbstractPluginModule {@Overridepublic List<CodeProblem> getCodeProblems(Build build, String blobPath, String reportName) {- return LockUtils.read(build.getReportCategoryLockKey(JobCheckstyleReport.DIR), new Callable<List<CodeProblem>>() {+ return LockUtils.read(build.getReportCategoryLockKey(PublishCheckstyleReportStep.DIR), new Callable<List<CodeProblem>>() {@SuppressWarnings("unchecked")@Overridepublic List<CodeProblem> call() throws Exception {List<CodeProblem> problems = new ArrayList<>();- File categoryDir = build.getReportCategoryDir(JobCheckstyleReport.DIR);+ File categoryDir = build.getReportCategoryDir(PublishCheckstyleReportStep.DIR);if (categoryDir.exists()) {for (File reportDir: categoryDir.listFiles()) {if (SecurityUtils.canAccessReport(build, reportDir.getName())&& (reportName == null || reportName.equals(reportDir.getName()))) {- File violationsFile = new File(reportDir, JobCheckstyleReport.VIOLATION_FILES + "/" + blobPath);+ File violationsFile = new File(reportDir, PublishCheckstyleReportStep.VIOLATION_FILES + "/" + blobPath);if (violationsFile.exists()) {try (InputStream is = new BufferedInputStream(new FileInputStream(violationsFile))) {for (ViolationFile.Violation violation:@@ -126,12 +126,12 @@ public class CheckstylePluginModule extends AbstractPluginModule {@Overridepublic List<BuildTab> getTabs(Build build) {List<BuildTab> tabs = new ArrayList<>();- LockUtils.read(build.getReportCategoryLockKey(JobCheckstyleReport.DIR), new Callable<Void>() {+ LockUtils.read(build.getReportCategoryLockKey(PublishCheckstyleReportStep.DIR), new Callable<Void>() {@Overridepublic Void call() throws Exception {- if (build.getReportCategoryDir(JobCheckstyleReport.DIR).exists()) {- for (File reportDir: build.getReportCategoryDir(JobCheckstyleReport.DIR).listFiles()) {+ if (build.getReportCategoryDir(PublishCheckstyleReportStep.DIR).exists()) {+ for (File reportDir: build.getReportCategoryDir(PublishCheckstyleReportStep.DIR).listFiles()) {if (!reportDir.isHidden() && SecurityUtils.canAccessReport(build, reportDir.getName())) {tabs.add(new BuildReportTab(reportDir.getName(), CheckstyleFilesPage.class,CheckstyleRulesPage.class, CheckstyleStatsPage.class));diff --git a/server-plugin/server-plugin-report-checkstyle/src/main/java/io/onedev/server/plugin/report/checkstyle/CheckstyleReportPage.java b/server-plugin/server-plugin-report-checkstyle/src/main/java/io/onedev/server/plugin/report/checkstyle/CheckstyleReportPage.javaindex e56ef39028..e39e24d1f2 100644--- a/server-plugin/server-plugin-report-checkstyle/src/main/java/io/onedev/server/plugin/report/checkstyle/CheckstyleReportPage.java+++ b/server-plugin/server-plugin-report-checkstyle/src/main/java/io/onedev/server/plugin/report/checkstyle/CheckstyleReportPage.java@@ -37,11 +37,11 @@ public class CheckstyleReportPage extends BuildReportPage {@Overrideprotected CheckstyleReportData load() {- return LockUtils.read(getBuild().getReportCategoryLockKey(JobCheckstyleReport.DIR), new Callable<CheckstyleReportData>() {+ return LockUtils.read(getBuild().getReportCategoryLockKey(PublishCheckstyleReportStep.DIR), new Callable<CheckstyleReportData>() {@Overridepublic CheckstyleReportData call() throws Exception {- File reportsDir = new File(getBuild().getReportCategoryDir(JobCheckstyleReport.DIR), getReportName());+ File reportsDir = new File(getBuild().getReportCategoryDir(PublishCheckstyleReportStep.DIR), getReportName());return CheckstyleReportData.readFrom(reportsDir);}diff --git a/server-plugin/server-plugin-report-checkstyle/src/main/java/io/onedev/server/plugin/report/checkstyle/JobCheckstyleReport.java b/server-plugin/server-plugin-report-checkstyle/src/main/java/io/onedev/server/plugin/report/checkstyle/PublishCheckstyleReportStep.javasimilarity index 90%rename from server-plugin/server-plugin-report-checkstyle/src/main/java/io/onedev/server/plugin/report/checkstyle/JobCheckstyleReport.javarename to server-plugin/server-plugin-report-checkstyle/src/main/java/io/onedev/server/plugin/report/checkstyle/PublishCheckstyleReportStep.javaindex 525e3a682b..34baa41225 100644--- a/server-plugin/server-plugin-report-checkstyle/src/main/java/io/onedev/server/plugin/report/checkstyle/JobCheckstyleReport.java+++ b/server-plugin/server-plugin-report-checkstyle/src/main/java/io/onedev/server/plugin/report/checkstyle/PublishCheckstyleReportStep.java@@ -20,7 +20,7 @@ import io.onedev.commons.utils.FileUtils;import io.onedev.commons.utils.LockUtils;import io.onedev.server.OneDev;import io.onedev.server.buildspec.BuildSpec;-import io.onedev.server.buildspec.job.JobReport;+import io.onedev.server.buildspec.step.PublishReportStep;import io.onedev.server.code.CodeProblem.Severity;import io.onedev.server.model.Build;import io.onedev.server.model.CheckstyleMetric;@@ -30,8 +30,8 @@ import io.onedev.server.web.editable.annotation.Editable;import io.onedev.server.web.editable.annotation.Interpolative;import io.onedev.server.web.editable.annotation.Patterns;-@Editable(name="Checkstyle Report")-public class JobCheckstyleReport extends JobReport {+@Editable(order=310, name="Publish Checkstyle Report")+public class PublishCheckstyleReportStep extends PublishReportStep {private static final long serialVersionUID = 1L;@@ -62,14 +62,14 @@ public class JobCheckstyleReport extends JobReport {}@Override- public void process(Build build, File workspace, SimpleLogger logger) {+ public void run(Build build, File filesDir, SimpleLogger logger) {File reportDir = new File(build.getReportCategoryDir(DIR), getReportName());CheckstyleReportData reportData = LockUtils.write(build.getReportCategoryLockKey(DIR), new Callable<CheckstyleReportData>() {@Overridepublic CheckstyleReportData call() throws Exception {- int baseLen = workspace.getAbsolutePath().length() + 1;+ int baseLen = filesDir.getAbsolutePath().length() + 1;SAXReader reader = new SAXReader();// Prevent XXE attack as the xml might be provided by malicious users@@ -78,7 +78,7 @@ public class JobCheckstyleReport extends JobReport {List<CheckstyleViolation> violations = new ArrayList<>();boolean hasReports = false;- for (File file: getPatternSet().listFiles(workspace)) {+ for (File file: getPatternSet().listFiles(filesDir)) {logger.log("Processing checkstyle report: " + file.getAbsolutePath().substring(baseLen));Document doc = reader.read(file);for (Element fileElement: doc.getRootElement().elements("file")) {diff --git a/server-plugin/server-plugin-report-checkstyle/src/main/java/io/onedev/server/plugin/report/checkstyle/JobESLintReport.java b/server-plugin/server-plugin-report-checkstyle/src/main/java/io/onedev/server/plugin/report/checkstyle/PublishESLintReportStep.javasimilarity index 84%rename from server-plugin/server-plugin-report-checkstyle/src/main/java/io/onedev/server/plugin/report/checkstyle/JobESLintReport.javarename to server-plugin/server-plugin-report-checkstyle/src/main/java/io/onedev/server/plugin/report/checkstyle/PublishESLintReportStep.javaindex 6d0be761a7..124e6da2c8 100644--- a/server-plugin/server-plugin-report-checkstyle/src/main/java/io/onedev/server/plugin/report/checkstyle/JobESLintReport.java+++ b/server-plugin/server-plugin-report-checkstyle/src/main/java/io/onedev/server/plugin/report/checkstyle/PublishESLintReportStep.java@@ -6,8 +6,8 @@ import io.onedev.server.web.editable.annotation.Editable;import io.onedev.server.web.editable.annotation.Interpolative;import io.onedev.server.web.editable.annotation.Patterns;-@Editable(name="ESLint Report")-public class JobESLintReport extends JobCheckstyleReport {+@Editable(order=300, name="Publish ESLint Report")+public class PublishESLintReportStep extends PublishCheckstyleReportStep {private static final long serialVersionUID = 1L;diff --git a/server-plugin/server-plugin-report-clover/src/main/java/io/onedev/server/plugin/report/clover/CloverPluginModule.java b/server-plugin/server-plugin-report-clover/src/main/java/io/onedev/server/plugin/report/clover/CloverPluginModule.javaindex a6667c4e95..a747972044 100644--- a/server-plugin/server-plugin-report-clover/src/main/java/io/onedev/server/plugin/report/clover/CloverPluginModule.java+++ b/server-plugin/server-plugin-report-clover/src/main/java/io/onedev/server/plugin/report/clover/CloverPluginModule.java@@ -21,7 +21,7 @@ import io.onedev.commons.launcher.loader.AbstractPluginModule;import io.onedev.commons.launcher.loader.ImplementationProvider;import io.onedev.commons.utils.LockUtils;import io.onedev.server.OneDev;-import io.onedev.server.buildspec.job.JobReport;+import io.onedev.server.buildspec.step.PublishReportStep;import io.onedev.server.code.LineCoverageContribution;import io.onedev.server.entitymanager.BuildMetricManager;import io.onedev.server.model.Build;@@ -52,12 +52,12 @@ public class CloverPluginModule extends AbstractPluginModule {@Overridepublic Class<?> getAbstractClass() {- return JobReport.class;+ return PublishReportStep.class;}@Overridepublic Collection<Class<?>> getImplementations() {- return Sets.newHashSet(JobJestCoverageReport.class, JobCloverReport.class);+ return Sets.newHashSet(PublishJestCoverageReportStep.class, PublishCloverReportStep.class);}});@@ -87,17 +87,17 @@ public class CloverPluginModule extends AbstractPluginModule {@Overridepublic Map<Integer, Integer> getLineCoverages(Build build, String blobPath, String reportName) {- return LockUtils.read(build.getReportCategoryLockKey(JobCloverReport.DIR), new Callable<Map<Integer, Integer>>() {+ return LockUtils.read(build.getReportCategoryLockKey(PublishCloverReportStep.DIR), new Callable<Map<Integer, Integer>>() {@Overridepublic Map<Integer, Integer> call() throws Exception {Map<Integer, Integer> coverages = new HashMap<>();- File categoryDir = build.getReportCategoryDir(JobCloverReport.DIR);+ File categoryDir = build.getReportCategoryDir(PublishCloverReportStep.DIR);if (categoryDir.exists()) {for (File reportDir: categoryDir.listFiles()) {if (SecurityUtils.canAccessReport(build, reportDir.getName())&& (reportName == null || reportName.equals(reportDir.getName()))) {- File testCountFile = new File(reportDir, JobCloverReport.TEST_COUNTS_DIR + "/" + blobPath);+ File testCountFile = new File(reportDir, PublishCloverReportStep.TEST_COUNTS_DIR + "/" + blobPath);if (testCountFile.exists()) {try (InputStream is = new BufferedInputStream(new FileInputStream(testCountFile))) {@SuppressWarnings("unchecked")@@ -124,12 +124,12 @@ public class CloverPluginModule extends AbstractPluginModule {@Overridepublic List<BuildTab> getTabs(Build build) {List<BuildTab> tabs = new ArrayList<>();- LockUtils.read(build.getReportCategoryLockKey(JobCloverReport.DIR), new Callable<Void>() {+ LockUtils.read(build.getReportCategoryLockKey(PublishCloverReportStep.DIR), new Callable<Void>() {@Overridepublic Void call() throws Exception {- if (build.getReportCategoryDir(JobCloverReport.DIR).exists()) {- for (File reportDir: build.getReportCategoryDir(JobCloverReport.DIR).listFiles()) {+ if (build.getReportCategoryDir(PublishCloverReportStep.DIR).exists()) {+ for (File reportDir: build.getReportCategoryDir(PublishCloverReportStep.DIR).listFiles()) {if (!reportDir.isHidden() && SecurityUtils.canAccessReport(build, reportDir.getName())) {tabs.add(new BuildReportTab(reportDir.getName(), CloverReportPage.class,CloverStatsPage.class));diff --git a/server-plugin/server-plugin-report-clover/src/main/java/io/onedev/server/plugin/report/clover/CloverReportPage.java b/server-plugin/server-plugin-report-clover/src/main/java/io/onedev/server/plugin/report/clover/CloverReportPage.javaindex 91c3ff3dd2..a109b29617 100644--- a/server-plugin/server-plugin-report-clover/src/main/java/io/onedev/server/plugin/report/clover/CloverReportPage.java+++ b/server-plugin/server-plugin-report-clover/src/main/java/io/onedev/server/plugin/report/clover/CloverReportPage.java@@ -82,12 +82,12 @@ public class CloverReportPage extends BuildReportPage {@Overrideprotected CloverReportData load() {- return LockUtils.read(getBuild().getReportCategoryLockKey(JobCloverReport.DIR), new Callable<CloverReportData>() {+ return LockUtils.read(getBuild().getReportCategoryLockKey(PublishCloverReportStep.DIR), new Callable<CloverReportData>() {@Overridepublic CloverReportData call() throws Exception {return CloverReportData.readFrom(new File(- getBuild().getReportCategoryDir(JobCloverReport.DIR), getReportName()));+ getBuild().getReportCategoryDir(PublishCloverReportStep.DIR), getReportName()));}});diff --git a/server-plugin/server-plugin-report-clover/src/main/java/io/onedev/server/plugin/report/clover/JobCloverReport.java b/server-plugin/server-plugin-report-clover/src/main/java/io/onedev/server/plugin/report/clover/PublishCloverReportStep.javasimilarity index 93%rename from server-plugin/server-plugin-report-clover/src/main/java/io/onedev/server/plugin/report/clover/JobCloverReport.javarename to server-plugin/server-plugin-report-clover/src/main/java/io/onedev/server/plugin/report/clover/PublishCloverReportStep.javaindex 49d0598c45..d547c63c0f 100644--- a/server-plugin/server-plugin-report-clover/src/main/java/io/onedev/server/plugin/report/clover/JobCloverReport.java+++ b/server-plugin/server-plugin-report-clover/src/main/java/io/onedev/server/plugin/report/clover/PublishCloverReportStep.java@@ -22,7 +22,7 @@ import io.onedev.commons.utils.FileUtils;import io.onedev.commons.utils.LockUtils;import io.onedev.server.OneDev;import io.onedev.server.buildspec.BuildSpec;-import io.onedev.server.buildspec.job.JobReport;+import io.onedev.server.buildspec.step.PublishReportStep;import io.onedev.server.model.Build;import io.onedev.server.model.CloverMetric;import io.onedev.server.persistence.dao.Dao;@@ -32,8 +32,8 @@ import io.onedev.server.web.editable.annotation.Editable;import io.onedev.server.web.editable.annotation.Interpolative;import io.onedev.server.web.editable.annotation.Patterns;-@Editable(name="Clover Coverage Report")-public class JobCloverReport extends JobReport {+@Editable(order=410, name="Publish Clover Coverage Report")+public class PublishCloverReportStep extends PublishReportStep {private static final long serialVersionUID = 1L;@@ -64,14 +64,14 @@ public class JobCloverReport extends JobReport {}@Override- public void process(Build build, File workspace, SimpleLogger logger) {+ public void run(Build build, File filesDir, SimpleLogger logger) {File reportDir = new File(build.getReportCategoryDir(DIR), getReportName());CloverReportData reportData = LockUtils.write(build.getReportCategoryLockKey(DIR), new Callable<CloverReportData>() {@Overridepublic CloverReportData call() throws Exception {- int baseLen = workspace.getAbsolutePath().length() + 1;+ int baseLen = filesDir.getAbsolutePath().length() + 1;SAXReader reader = new SAXReader();// Prevent XXE attack as the xml might be provided by malicious users@@ -88,7 +88,7 @@ public class JobCloverReport extends JobReport {List<PackageCoverageInfo> packageCoverages = new ArrayList<>();- for (File file: getPatternSet().listFiles(workspace)) {+ for (File file: getPatternSet().listFiles(filesDir)) {logger.log("Processing clover report: " + file.getAbsolutePath().substring(baseLen));Document doc = reader.read(file);for (Element projectElement: doc.getRootElement().elements("project")) {diff --git a/server-plugin/server-plugin-report-clover/src/main/java/io/onedev/server/plugin/report/clover/JobJestCoverageReport.java b/server-plugin/server-plugin-report-clover/src/main/java/io/onedev/server/plugin/report/clover/PublishJestCoverageReportStep.javasimilarity index 84%rename from server-plugin/server-plugin-report-clover/src/main/java/io/onedev/server/plugin/report/clover/JobJestCoverageReport.javarename to server-plugin/server-plugin-report-clover/src/main/java/io/onedev/server/plugin/report/clover/PublishJestCoverageReportStep.javaindex ad68d37af5..0fd451336e 100644--- a/server-plugin/server-plugin-report-clover/src/main/java/io/onedev/server/plugin/report/clover/JobJestCoverageReport.java+++ b/server-plugin/server-plugin-report-clover/src/main/java/io/onedev/server/plugin/report/clover/PublishJestCoverageReportStep.java@@ -6,8 +6,8 @@ import io.onedev.server.web.editable.annotation.Editable;import io.onedev.server.web.editable.annotation.Interpolative;import io.onedev.server.web.editable.annotation.Patterns;-@Editable(name="Jest Coverage Report")-public class JobJestCoverageReport extends JobCloverReport {+@Editable(order=400, name="Publish Jest Coverage Report")+public class PublishJestCoverageReportStep extends PublishCloverReportStep {private static final long serialVersionUID = 1L;diff --git a/server-plugin/server-plugin-report-jest/src/main/java/io/onedev/server/plugin/report/jest/JestTestReportModule.java b/server-plugin/server-plugin-report-jest/src/main/java/io/onedev/server/plugin/report/jest/JestTestReportModule.javaindex 6f55261884..1c8b54bc52 100644--- a/server-plugin/server-plugin-report-jest/src/main/java/io/onedev/server/plugin/report/jest/JestTestReportModule.java+++ b/server-plugin/server-plugin-report-jest/src/main/java/io/onedev/server/plugin/report/jest/JestTestReportModule.java@@ -15,7 +15,7 @@ import io.onedev.commons.launcher.loader.AbstractPluginModule;import io.onedev.commons.launcher.loader.ImplementationProvider;import io.onedev.commons.utils.LockUtils;import io.onedev.server.OneDev;-import io.onedev.server.buildspec.job.JobReport;+import io.onedev.server.buildspec.step.PublishReportStep;import io.onedev.server.entitymanager.BuildMetricManager;import io.onedev.server.model.Build;import io.onedev.server.model.JestTestMetric;@@ -45,12 +45,12 @@ public class JestTestReportModule extends AbstractPluginModule {@Overridepublic Class<?> getAbstractClass() {- return JobReport.class;+ return PublishReportStep.class;}@Overridepublic Collection<Class<?>> getImplementations() {- return Sets.newHashSet(JobJestTestReport.class);+ return Sets.newHashSet(PublishJestTestReportStep.class);}});@@ -60,11 +60,11 @@ public class JestTestReportModule extends AbstractPluginModule {@Overridepublic List<BuildTab> getTabs(Build build) {List<BuildTab> tabs = new ArrayList<>();- LockUtils.read(build.getReportCategoryLockKey(JobJestTestReport.DIR), new Callable<Void>() {+ LockUtils.read(build.getReportCategoryLockKey(PublishJestTestReportStep.DIR), new Callable<Void>() {@Overridepublic Void call() throws Exception {- File categoryDir = build.getReportCategoryDir(JobJestTestReport.DIR);+ File categoryDir = build.getReportCategoryDir(PublishJestTestReportStep.DIR);if (categoryDir.exists()) {for (File reportDir: categoryDir.listFiles()) {if (!reportDir.isHidden() && SecurityUtils.canAccessReport(build, reportDir.getName())) {diff --git a/server-plugin/server-plugin-report-jest/src/main/java/io/onedev/server/plugin/report/jest/JestTestReportPage.java b/server-plugin/server-plugin-report-jest/src/main/java/io/onedev/server/plugin/report/jest/JestTestReportPage.javaindex d6d3914b75..9ab5e6542c 100644--- a/server-plugin/server-plugin-report-jest/src/main/java/io/onedev/server/plugin/report/jest/JestTestReportPage.java+++ b/server-plugin/server-plugin-report-jest/src/main/java/io/onedev/server/plugin/report/jest/JestTestReportPage.java@@ -32,11 +32,11 @@ public abstract class JestTestReportPage extends BuildReportPage {@Overrideprotected JestTestReportData load() {- return LockUtils.read(getBuild().getReportCategoryLockKey(JobJestTestReport.DIR), new Callable<JestTestReportData>() {+ return LockUtils.read(getBuild().getReportCategoryLockKey(PublishJestTestReportStep.DIR), new Callable<JestTestReportData>() {@Overridepublic JestTestReportData call() throws Exception {- File reportsDir = new File(getBuild().getReportCategoryDir(JobJestTestReport.DIR), getReportName());+ File reportsDir = new File(getBuild().getReportCategoryDir(PublishJestTestReportStep.DIR), getReportName());return JestTestReportData.readFrom(reportsDir);}diff --git a/server-plugin/server-plugin-report-jest/src/main/java/io/onedev/server/plugin/report/jest/JobJestTestReport.java b/server-plugin/server-plugin-report-jest/src/main/java/io/onedev/server/plugin/report/jest/PublishJestTestReportStep.javasimilarity index 87%rename from server-plugin/server-plugin-report-jest/src/main/java/io/onedev/server/plugin/report/jest/JobJestTestReport.javarename to server-plugin/server-plugin-report-jest/src/main/java/io/onedev/server/plugin/report/jest/PublishJestTestReportStep.javaindex 2ee58bd991..ab207bb41a 100644--- a/server-plugin/server-plugin-report-jest/src/main/java/io/onedev/server/plugin/report/jest/JobJestTestReport.java+++ b/server-plugin/server-plugin-report-jest/src/main/java/io/onedev/server/plugin/report/jest/PublishJestTestReportStep.java@@ -17,7 +17,7 @@ import io.onedev.commons.utils.FileUtils;import io.onedev.commons.utils.LockUtils;import io.onedev.server.OneDev;import io.onedev.server.buildspec.BuildSpec;-import io.onedev.server.buildspec.job.JobReport;+import io.onedev.server.buildspec.step.PublishReportStep;import io.onedev.server.model.Build;import io.onedev.server.model.JestTestMetric;import io.onedev.server.persistence.dao.Dao;@@ -26,8 +26,8 @@ import io.onedev.server.web.editable.annotation.Editable;import io.onedev.server.web.editable.annotation.Interpolative;import io.onedev.server.web.editable.annotation.Patterns;-@Editable(name="Jest Test Report")-public class JobJestTestReport extends JobReport {+@Editable(order=300, name="Publish Jest Test Report")+public class PublishJestTestReportStep extends PublishReportStep {private static final long serialVersionUID = 1L;@@ -54,7 +54,7 @@ public class JobJestTestReport extends JobReport {}@Override- public void process(Build build, File workspace, SimpleLogger logger) {+ public void run(Build build, File filesDir, SimpleLogger logger) {File reportDir = new File(build.getReportCategoryDir(DIR), getReportName());JestTestReportData report = LockUtils.write(build.getReportCategoryLockKey(DIR), new Callable<JestTestReportData>() {@@ -64,8 +64,8 @@ public class JobJestTestReport extends JobReport {ObjectMapper mapper = OneDev.getInstance(ObjectMapper.class);Collection<JsonNode> rootNodes = new ArrayList<>();- int baseLen = workspace.getAbsolutePath().length()+1;- for (File file: getPatternSet().listFiles(workspace)) {+ int baseLen = filesDir.getAbsolutePath().length()+1;+ for (File file: filesDir.listFiles()) {logger.log("Processing jest test report: " + file.getAbsolutePath().substring(baseLen));try {rootNodes.add(mapper.readTree(file));@@ -95,8 +95,7 @@ public class JobJestTestReport extends JobReport {metric.setNumOfTestSuites(report.getNumOfTestSuites());metric.setTotalTestDuration(report.getTotalTestDuration());OneDev.getInstance(Dao.class).persist(metric);- }-+ }}}diff --git a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/MarkdownReportDownloadResource.java b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/MarkdownReportDownloadResource.javaindex 54159c9447..51062e6fff 100644--- a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/MarkdownReportDownloadResource.java+++ b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/MarkdownReportDownloadResource.java@@ -94,7 +94,7 @@ public class MarkdownReportDownloadResource extends AbstractResource {String markdownPath = Joiner.on("/").join(pathSegments);File buildDir = OneDev.getInstance(StorageManager.class).getBuildDir(project.getId(), build.getNumber());- File reportDir = new File(buildDir, JobMarkdownReport.DIR + "/" + reportName);+ File reportDir = new File(buildDir, PublishMarkdownReportStep.DIR + "/" + reportName);File markdownFile = new File(reportDir, markdownPath);if (!markdownFile.exists() || markdownFile.isDirectory()) {@@ -121,7 +121,7 @@ public class MarkdownReportDownloadResource extends AbstractResource {@Overridepublic void writeData(Attributes attributes) throws IOException {- LockUtils.read(build.getReportCategoryLockKey(JobMarkdownReport.DIR), new Callable<Void>() {+ LockUtils.read(build.getReportCategoryLockKey(PublishMarkdownReportStep.DIR), new Callable<Void>() {@Overridepublic Void call() throws Exception {diff --git a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/MarkdownReportModule.java b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/MarkdownReportModule.javaindex fa5b554cef..b4c1e4696a 100644--- a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/MarkdownReportModule.java+++ b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/MarkdownReportModule.java@@ -13,7 +13,7 @@ import com.google.common.collect.Sets;import io.onedev.commons.launcher.loader.AbstractPluginModule;import io.onedev.commons.launcher.loader.ImplementationProvider;import io.onedev.commons.utils.LockUtils;-import io.onedev.server.buildspec.job.JobReport;+import io.onedev.server.buildspec.step.PublishReportStep;import io.onedev.server.model.Build;import io.onedev.server.model.PullRequest;import io.onedev.server.security.SecurityUtils;@@ -38,12 +38,12 @@ public class MarkdownReportModule extends AbstractPluginModule {@Overridepublic Class<?> getAbstractClass() {- return JobReport.class;+ return PublishReportStep.class;}@Overridepublic Collection<Class<?>> getImplementations() {- return Sets.newHashSet(JobMarkdownReport.class, JobPullRequestMarkdownReport.class);+ return Sets.newHashSet(PublishMarkdownReportStep.class, PublishPullRequestMarkdownReportStep.class);}});@@ -53,11 +53,11 @@ public class MarkdownReportModule extends AbstractPluginModule {@Overridepublic List<BuildTab> getTabs(Build build) {List<BuildTab> tabs = new ArrayList<>();- LockUtils.read(build.getReportCategoryLockKey(JobMarkdownReport.DIR), new Callable<Void>() {+ LockUtils.read(build.getReportCategoryLockKey(PublishMarkdownReportStep.DIR), new Callable<Void>() {@Overridepublic Void call() throws Exception {- File categoryDir = build.getReportCategoryDir(JobMarkdownReport.DIR);+ File categoryDir = build.getReportCategoryDir(PublishMarkdownReportStep.DIR);if (categoryDir.exists()) {for (File reportDir: categoryDir.listFiles()) {if (SecurityUtils.canAccessReport(build, reportDir.getName()))@@ -84,12 +84,12 @@ public class MarkdownReportModule extends AbstractPluginModule {public List<PullRequestSummaryPart> getParts(PullRequest request) {List<PullRequestSummaryPart> parts = new ArrayList<>();for (Build build: request.getCurrentBuilds()) {- parts.addAll(LockUtils.read(build.getReportCategoryLockKey(JobMarkdownReport.DIR), new Callable<List<PullRequestSummaryPart>>() {+ parts.addAll(LockUtils.read(build.getReportCategoryLockKey(PublishMarkdownReportStep.DIR), new Callable<List<PullRequestSummaryPart>>() {@Overridepublic List<PullRequestSummaryPart> call() throws Exception {List<PullRequestSummaryPart> parts = new ArrayList<>();- File categoryDir = build.getReportCategoryDir(JobPullRequestMarkdownReport.DIR);+ File categoryDir = build.getReportCategoryDir(PublishPullRequestMarkdownReportStep.DIR);if (categoryDir.exists()) {for (File reportDir: categoryDir.listFiles()) {if (SecurityUtils.canAccessReport(build, reportDir.getName()))diff --git a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/MarkdownReportPage.java b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/MarkdownReportPage.javaindex 52e7a7b8e2..d5211755fb 100644--- a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/MarkdownReportPage.java+++ b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/MarkdownReportPage.java@@ -61,7 +61,7 @@ public class MarkdownReportPage extends BuildDetailPage {protected void onInitialize() {super.onInitialize();- File file = new File(getBuild().getReportCategoryDir(JobMarkdownReport.DIR),+ File file = new File(getBuild().getReportCategoryDir(PublishMarkdownReportStep.DIR),reportName + "/" + filePath);try {String markdown = FileUtils.readFileToString(file, StandardCharsets.UTF_8);diff --git a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/MarkdownReportTab.java b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/MarkdownReportTab.javaindex 8dd53e6116..8c52ff3b82 100644--- a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/MarkdownReportTab.java+++ b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/MarkdownReportTab.java@@ -30,12 +30,12 @@ public class MarkdownReportTab extends BuildTab {@Overrideprotected Link<?> newLink(String linkId, Class<? extends Page> pageClass) {BuildDetailPage page = (BuildDetailPage) getPage();- String startPage = LockUtils.read(page.getBuild().getReportCategoryLockKey(JobMarkdownReport.DIR), new Callable<String>() {+ String startPage = LockUtils.read(page.getBuild().getReportCategoryLockKey(PublishMarkdownReportStep.DIR), new Callable<String>() {@Overridepublic String call() throws Exception {- File startPageFile = new File(page.getBuild().getReportCategoryDir(JobMarkdownReport.DIR),- getTitle() + "/" + JobMarkdownReport.START_PAGE);+ File startPageFile = new File(page.getBuild().getReportCategoryDir(PublishMarkdownReportStep.DIR),+ getTitle() + "/" + PublishMarkdownReportStep.START_PAGE);return FileUtils.readFileToString(startPageFile, StandardCharsets.UTF_8);}diff --git a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/JobMarkdownReport.java b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PublishMarkdownReportStep.javasimilarity index 80%rename from server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/JobMarkdownReport.javarename to server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PublishMarkdownReportStep.javaindex 1b6096356e..f198be6313 100644--- a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/JobMarkdownReport.java+++ b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PublishMarkdownReportStep.java@@ -11,14 +11,14 @@ import io.onedev.commons.codeassist.InputSuggestion;import io.onedev.commons.utils.FileUtils;import io.onedev.commons.utils.LockUtils;import io.onedev.server.buildspec.BuildSpec;-import io.onedev.server.buildspec.job.JobReport;+import io.onedev.server.buildspec.step.PublishReportStep;import io.onedev.server.model.Build;import io.onedev.server.util.SimpleLogger;import io.onedev.server.web.editable.annotation.Editable;import io.onedev.server.web.editable.annotation.Interpolative;-@Editable(name="Markdown Report")-public class JobMarkdownReport extends JobReport {+@Editable(order=110, name="Publish Markdown Report")+public class PublishMarkdownReportStep extends PublishReportStep {private static final long serialVersionUID = 1L;@@ -45,21 +45,21 @@ public class JobMarkdownReport extends JobReport {}@Override- public void process(Build build, File workspace, SimpleLogger logger) {+ public void run(Build build, File filesDir, SimpleLogger logger) {File reportDir = new File(build.getReportCategoryDir(DIR), getReportName());LockUtils.write(build.getReportCategoryLockKey(DIR), new Callable<Void>() {@Overridepublic Void call() throws Exception {- File startPage = new File(workspace, getStartPage());+ File startPage = new File(filesDir, getStartPage());if (startPage.exists()) {FileUtils.createDir(reportDir);File startPageFile = new File(reportDir, START_PAGE);FileUtils.writeFile(startPageFile, getStartPage());- int baseLen = workspace.getAbsolutePath().length() + 1;- for (File file: getPatternSet().listFiles(workspace)) {+ int baseLen = filesDir.getAbsolutePath().length() + 1;+ for (File file: getPatternSet().listFiles(filesDir)) {try {FileUtils.copyFile(file, new File(reportDir, file.getAbsolutePath().substring(baseLen)));} catch (IOException e) {diff --git a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/JobPullRequestMarkdownReport.java b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PublishPullRequestMarkdownReportStep.javasimilarity index 83%rename from server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/JobPullRequestMarkdownReport.javarename to server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PublishPullRequestMarkdownReportStep.javaindex 7de92b5374..0e47bb7c28 100644--- a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/JobPullRequestMarkdownReport.java+++ b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PublishPullRequestMarkdownReportStep.java@@ -11,15 +11,15 @@ import io.onedev.commons.codeassist.InputSuggestion;import io.onedev.commons.utils.FileUtils;import io.onedev.commons.utils.LockUtils;import io.onedev.server.buildspec.BuildSpec;-import io.onedev.server.buildspec.job.JobReport;+import io.onedev.server.buildspec.step.PublishReportStep;import io.onedev.server.model.Build;import io.onedev.server.util.SimpleLogger;import io.onedev.server.web.editable.annotation.Editable;import io.onedev.server.web.editable.annotation.Interpolative;-@Editable(name="Markdown Report (for Pull Request)", description="This report will be displayed in pull request overview "+@Editable(order=120, name="Publish Markdown Report (for Pull Request)", description="This report will be displayed in pull request overview "+ "page if the build belongs to a pull request")-public class JobPullRequestMarkdownReport extends JobReport {+public class PublishPullRequestMarkdownReportStep extends PublishReportStep {private static final long serialVersionUID = 1L;@@ -51,7 +51,7 @@ public class JobPullRequestMarkdownReport extends JobReport {}@Override- public void process(Build build, File workspace, SimpleLogger logger) {+ public void run(Build build, File workspace, SimpleLogger logger) {if (build.getRequest() != null) {File reportDir = new File(build.getReportCategoryDir(DIR), getReportName());diff --git a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PullRequestSummaryMarkdownPart.java b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PullRequestSummaryMarkdownPart.javaindex f9c9eda50a..22caa1322b 100644--- a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PullRequestSummaryMarkdownPart.java+++ b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PullRequestSummaryMarkdownPart.java@@ -24,7 +24,7 @@ public class PullRequestSummaryMarkdownPart extends PullRequestSummaryPart {@Overridepublic Component render(String componentId) {try {- File file = new File(reportDir, JobPullRequestMarkdownReport.FILE);+ File file = new File(reportDir, PublishPullRequestMarkdownReportStep.FILE);String markdown = FileUtils.readFileToString(file, StandardCharsets.UTF_8);return new MarkdownViewer(componentId, Model.of(markdown), null).add(AttributeAppender.append("class", "mb-n3"));diff --git a/server-product/src/test/java/io/onedev/server/product/Test.java b/server-product/src/test/java/io/onedev/server/product/Test.javaindex 1bc98e632b..e29e84b6bf 100644--- a/server-product/src/test/java/io/onedev/server/product/Test.java+++ b/server-product/src/test/java/io/onedev/server/product/Test.java@@ -4,6 +4,6 @@ public class Test {@org.junit.Testpublic void test() {- }-+ }+}\ No newline at end of file