The Unified and Autonomous Development Platform
Run container step support for linux
Robin Shenba5575aa8266
diff --git a/doc/incompatibilities.md b/doc/incompatibilities.mdindex 5b386032e3..a16030df1b 100644--- a/doc/incompatibilities.md+++ b/doc/incompatibilities.md@@ -2,6 +2,7 @@1. A new property `Run In Container` is added and enabled for all command steps in build spec. If you have steps intended tobe executed by shell executor, edit it to disable this property, otherwise the build will fail reporting `This step should be executed by server docker executor, remote docker executor, or kubernetes executor`+1. Using cache path "." (to cache workspace itself) in job cache definition is no longer supported# 5.4.0diff --git a/server-core/src/main/java/io/onedev/server/buildspec/job/CacheSpec.java b/server-core/src/main/java/io/onedev/server/buildspec/job/CacheSpec.javaindex 3fe1352b76..19108e75f5 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/job/CacheSpec.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/job/CacheSpec.java@@ -3,16 +3,22 @@ package io.onedev.server.buildspec.job;import java.io.Serializable;import java.util.List;+import javax.validation.ConstraintValidatorContext;+import org.hibernate.validator.constraints.NotEmpty;import io.onedev.commons.codeassist.InputSuggestion;+import io.onedev.commons.utils.PathUtils;import io.onedev.server.buildspec.BuildSpec;+import io.onedev.server.util.validation.Validatable;+import io.onedev.server.util.validation.annotation.ClassValidating;import io.onedev.server.util.validation.annotation.RegEx;import io.onedev.server.web.editable.annotation.Editable;import io.onedev.server.web.editable.annotation.Interpolative;@Editable-public class CacheSpec implements Serializable {+@ClassValidating+public class CacheSpec implements Serializable, Validatable {private static final long serialVersionUID = 1L;@@ -20,7 +26,8 @@ public class CacheSpec implements Serializable {private String path;- @Editable(order=100, description="Specify key of the cache. Caches with same key can be reused by different jobs")+ @Editable(order=100, description="Specify key of the cache. Caches with same key can be reused by different projects/jobs. "+ + "Embed project/job variable to prevent cross project/job reuse")@Interpolative(variableSuggester="suggestVariables")@NotEmpty@RegEx(pattern="[a-zA-Z0-9\\-_\\.]+", message="Can only contain alphanumeric, dash, dot and underscore")@@ -33,8 +40,7 @@ public class CacheSpec implements Serializable {}@Editable(order=200, description="Specify path to cache. Non-absolute path is considered to be relative to job workspace. "- + "Specify \".\" (without quote) to cache workspace itself. "- + "<span class='text-warning'>Absolute path is not allowed if the job is executed by a shell executor</span>")+ + "Please note that shell executor only allows non-absolute path here")@Interpolative(variableSuggester="suggestVariables")@NotEmptypublic String getPath() {@@ -50,4 +56,15 @@ public class CacheSpec implements Serializable {return BuildSpec.suggestVariables(matchWith, false, false);}+ @Override+ public boolean isValid(ConstraintValidatorContext context) {+ if (PathUtils.isCurrent(path)) {+ context.disableDefaultConstraintViolation();+ context.buildConstraintViolationWithTemplate("Invalid path").addPropertyNode("path").addConstraintViolation();+ return false;+ } else {+ return true;+ }+ }+}diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/CheckoutStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/CheckoutStep.javaindex 6baf9f959e..148bb2007a 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/step/CheckoutStep.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/CheckoutStep.java@@ -10,7 +10,7 @@ import io.onedev.server.buildspec.param.ParamCombination;import io.onedev.server.model.Build;import io.onedev.server.web.editable.annotation.Editable;-@Editable(order=5, name="Checkout Code")+@Editable(order=50, name="Checkout Code")public class CheckoutStep extends Step {private static final long serialVersionUID = 1L;diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/CloseMilestoneStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/CloseMilestoneStep.javaindex dcd9ba96ae..b3df2e0da7 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/step/CloseMilestoneStep.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/CloseMilestoneStep.java@@ -19,7 +19,7 @@ import io.onedev.server.persistence.TransactionManager;import io.onedev.server.web.editable.annotation.Editable;import io.onedev.server.web.editable.annotation.Interpolative;-@Editable(name="Close Milestone", order=40)+@Editable(name="Close Milestone", order=400)public class CloseMilestoneStep extends ServerStep {private static final long serialVersionUID = 1L;diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/CreateTagStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/CreateTagStep.javaindex cf5dfa8b0c..1d4498daa5 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/step/CreateTagStep.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/CreateTagStep.java@@ -21,7 +21,7 @@ import io.onedev.server.web.editable.annotation.Editable;import io.onedev.server.web.editable.annotation.Interpolative;import io.onedev.server.web.editable.annotation.Multiline;-@Editable(name="Create Tag", order=30)+@Editable(name="Create Tag", order=300)public class CreateTagStep extends ServerStep {private static final long serialVersionUID = 1L;diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/PublishArtifactStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/PublishArtifactStep.javaindex 6ed3dd139c..5417de8ddc 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/step/PublishArtifactStep.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/PublishArtifactStep.java@@ -18,7 +18,7 @@ import io.onedev.server.web.editable.annotation.Editable;import io.onedev.server.web.editable.annotation.Interpolative;import io.onedev.server.web.editable.annotation.Patterns;-@Editable(order=105, name="Publish Artifacts")+@Editable(order=1050, name="Publish Artifacts")public class PublishArtifactStep extends ServerStep {private static final long serialVersionUID = 1L;diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/RunContainerStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/RunContainerStep.javanew file mode 100644index 0000000000..080f2b06be--- /dev/null+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/RunContainerStep.java@@ -0,0 +1,108 @@+package io.onedev.server.buildspec.step;++import java.util.ArrayList;+import java.util.HashMap;+import java.util.List;+import java.util.Map;++import javax.annotation.Nullable;++import org.hibernate.validator.constraints.NotEmpty;++import io.onedev.commons.codeassist.InputSuggestion;+import io.onedev.k8shelper.ContainerExecutable;+import io.onedev.k8shelper.Executable;+import io.onedev.server.buildspec.BuildSpec;+import io.onedev.server.buildspec.job.EnvVar;+import io.onedev.server.buildspec.param.ParamCombination;+import io.onedev.server.model.Build;+import io.onedev.server.web.editable.annotation.Editable;+import io.onedev.server.web.editable.annotation.Interpolative;++@Editable(order=150, name="Run Container")+public class RunContainerStep extends Step {++ private static final long serialVersionUID = 1L;++ private String image;++ private String args;++ private List<EnvVar> envVars = new ArrayList<>();++ private String workingDir;++ private boolean useTTY;++ @Override+ public Executable getExecutable(Build build, String jobToken, ParamCombination paramCombination) {+ Map<String, String> envMap = new HashMap<>();+ for (EnvVar var: getEnvVars())+ envMap.put(var.getName(), var.getValue());+ return new ContainerExecutable(getImage(), getArgs(), envMap, getWorkingDir(), isUseTTY());+ }++ @Editable(order=100, description="Specify container image to run. <b class='text-warning'>NOTE:</b> A shell must "+ + "exist in the container if the step is executed by kubernetes executor, as OneDev intercepts the "+ + "entrypoint to make step containers executing sequentially in the pod")+ @Interpolative(variableSuggester="suggestVariables")+ @NotEmpty+ public String getImage() {+ return image;+ }++ public void setImage(String image) {+ this.image = image;+ }++ @Editable(order=200, name="Arguments", description="Specify container arguments separated by space. "+ + "Single argument containing space should be quoted")+ @Interpolative(variableSuggester="suggestVariables")+ public String getArgs() {+ return args;+ }++ public void setArgs(String args) {+ this.args = args;+ }++ @Editable(order=200, name="Working Directory", description="Optionally specify working directory of the container. "+ + "Leave empty to use the default working directory (<tt>/onedev-build/workspace</tt>). "+ + "<a href='$docRoot/pages/concepts.md#job-workspace' target='_blank'>Job workspace</a> will be "+ + "mounted to the working directory so that the container can access job files")+ @Nullable+ public String getWorkingDir() {+ return workingDir;+ }++ public void setWorkingDir(String workingDir) {+ this.workingDir = workingDir;+ }++ @Editable(order=400, name="Environment Variables", description="Optionally specify environment "+ + "variables for the container")+ public List<EnvVar> getEnvVars() {+ return envVars;+ }++ public void setEnvVars(List<EnvVar> envVars) {+ this.envVars = envVars;+ }++ @Editable(order=10000, name="Enable TTY Mode", description="Many commands print outputs with ANSI colors in "+ + "TTY mode to help identifying problems easily. However some commands running in this mode may "+ + "wait for user input to cause build hanging. This can normally be fixed by adding extra options "+ + "to the command")+ public boolean isUseTTY() {+ return useTTY;+ }++ public void setUseTTY(boolean useTTY) {+ this.useTTY = useTTY;+ }++ static List<InputSuggestion> suggestVariables(String matchWith) {+ return BuildSpec.suggestVariables(matchWith, false, false);+ }++}diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/SetBuildVersionStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/SetBuildVersionStep.javaindex 3f24fab0c7..7bd96b2d0f 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/step/SetBuildVersionStep.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/SetBuildVersionStep.java@@ -22,7 +22,7 @@ import io.onedev.server.persistence.TransactionManager;import io.onedev.server.web.editable.annotation.Editable;import io.onedev.server.web.editable.annotation.Interpolative;-@Editable(order=20, name="Set Build Version")+@Editable(order=260, name="Set Build Version")public class SetBuildVersionStep extends ServerStep {private static final long serialVersionUID = 1L;diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/command/CommandStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/command/CommandStep.javaindex 922f0d99bd..b7f24e81d6 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/step/command/CommandStep.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/command/CommandStep.java@@ -19,7 +19,7 @@ import io.onedev.server.web.editable.annotation.Editable;import io.onedev.server.web.editable.annotation.Interpolative;import io.onedev.server.web.editable.annotation.ShowCondition;-@Editable(order=10, name="Execute Commands")+@Editable(order=100, name="Execute Commands")public class CommandStep extends Step {private static final long serialVersionUID = 1L;diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/command/interpreter/BorneShellInterpreter.java b/server-core/src/main/java/io/onedev/server/buildspec/step/command/interpreter/BashInterpreter.javasimilarity index 75%rename from server-core/src/main/java/io/onedev/server/buildspec/step/command/interpreter/BorneShellInterpreter.javarename to server-core/src/main/java/io/onedev/server/buildspec/step/command/interpreter/BashInterpreter.javaindex 2ef75a7ef2..a2808f4f1e 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/step/command/interpreter/BorneShellInterpreter.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/command/interpreter/BashInterpreter.java@@ -4,18 +4,18 @@ import java.util.List;import javax.validation.constraints.Size;-import io.onedev.k8shelper.BorneShellExecutable;+import io.onedev.k8shelper.BashExecutable;import io.onedev.k8shelper.CommandExecutable;import io.onedev.server.util.validation.annotation.Code;import io.onedev.server.web.editable.annotation.Editable;import io.onedev.server.web.editable.annotation.Interpolative;-@Editable(order=200, name="Borne Shell")-public class BorneShellInterpreter extends Interpreter {+@Editable(order=200, name="bash")+public class BashInterpreter extends Interpreter {private static final long serialVersionUID = 1L;- @Editable(order=110, description="Specify borne shell commands to execute "+ @Editable(order=110, description="Specify bash commands to execute "+ "under the <a href='$docRoot/pages/concepts.md#job-workspace' target='_blank'>job workspace</a>")@Interpolative@Code(language=Code.SHELL, variableProvider="suggestVariables")@@ -30,7 +30,7 @@ public class BorneShellInterpreter extends Interpreter {@Overridepublic CommandExecutable getExecutable(String image, boolean useTTY) {- return new BorneShellExecutable(image, getCommands(), useTTY);+ return new BashExecutable(image, getCommands(), useTTY);}}diff --git a/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesExecutor.java b/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesExecutor.javaindex 77a193b926..2da094a22d 100644--- a/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesExecutor.java+++ b/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesExecutor.java@@ -56,6 +56,7 @@ import io.onedev.commons.utils.command.LineConsumer;import io.onedev.k8shelper.Action;import io.onedev.k8shelper.CommandExecutable;import io.onedev.k8shelper.CompositeExecutable;+import io.onedev.k8shelper.ContainerExecutable;import io.onedev.k8shelper.ExecuteCondition;import io.onedev.k8shelper.KubernetesHelper;import io.onedev.k8shelper.LeafExecutable;@@ -701,6 +702,7 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestDataString containerAuthInfoHome;String trustCertsHome;String dockerSock;+ String containerdSock;if (baselineOsInfo.isLinux()) {containerBuildHome = "/onedev-build";containerCacheHome = containerBuildHome + "/cache";@@ -708,6 +710,7 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestDatacontainerAuthInfoHome = "/root/auth-info";trustCertsHome = containerBuildHome + "/trust-certs";dockerSock = "/var/run/docker.sock";+ containerdSock = "/run/containerd/containerd.sock";} else {containerBuildHome = "C:\\onedev-build";containerCacheHome = containerBuildHome + "\\cache";@@ -715,6 +718,7 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestDatacontainerAuthInfoHome = "C:\\Users\\ContainerAdministrator\\auth-info";trustCertsHome = containerBuildHome + "\\trust-certs";dockerSock = null;+ containerdSock = null;}Map<String, String> buildHomeMount = CollectionUtils.newLinkedHashMap(@@ -738,14 +742,19 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestDataMap<String, String> dockerSockMount = CollectionUtils.newLinkedHashMap("name", "docker-sock","mountPath", dockerSock);+ Map<String, String> containerdSockMount = CollectionUtils.newLinkedHashMap(+ "name", "containerd-sock",+ "mountPath", containerdSock);- List<Object> volumeMounts = Lists.<Object>newArrayList(buildHomeMount, authInfoMount, cacheHomeMount);+ List<Object> commonVolumeMounts = Lists.<Object>newArrayList(buildHomeMount, authInfoMount, cacheHomeMount);if (baselineOsInfo.isWindows())- volumeMounts.add(authInfoMount2);+ commonVolumeMounts.add(authInfoMount2);if (trustCertsConfigMapName != null)- volumeMounts.add(trustCertsMount);+ commonVolumeMounts.add(trustCertsMount);if (dockerSock != null)- volumeMounts.add(dockerSockMount);+ commonVolumeMounts.add(dockerSockMount);+ if (containerdSock != null)+ commonVolumeMounts.add(containerdSockMount);CompositeExecutable entryExecutable;if (jobContext != null) {@@ -769,11 +778,11 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestDatathrow new RuntimeException(e);}- List<Map<Object, Object>> envs = new ArrayList<>();- envs.add(CollectionUtils.newLinkedHashMap(+ List<Map<Object, Object>> commonEnvs = new ArrayList<>();+ commonEnvs.add(CollectionUtils.newLinkedHashMap("name", ENV_SERVER_URL,"value", getServerUrl()));- envs.add(CollectionUtils.newLinkedHashMap(+ commonEnvs.add(CollectionUtils.newLinkedHashMap("name", ENV_JOB_TOKEN,"value", jobToken));@@ -787,19 +796,45 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestDataif (executable instanceof CommandExecutable) {CommandExecutable commandExecutable = (CommandExecutable) executable;if (commandExecutable.getImage() == null) {- throw new ExplicitException("This step should be executed by server shell "+ throw new ExplicitException("This step can only be executed by server shell "+ "executor or remote shell executor");}stepContainerSpec = CollectionUtils.newHashMap("name", containerName,"image", commandExecutable.getImage());- if (((CommandExecutable) executable).isUseTTY())+ if (commandExecutable.isUseTTY())stepContainerSpec.put("tty", true);+ stepContainerSpec.put("volumeMounts", commonVolumeMounts);+ stepContainerSpec.put("env", commonEnvs);+ } else if (executable instanceof ContainerExecutable) {+ ContainerExecutable containerExecutable = (ContainerExecutable) executable;+ stepContainerSpec = CollectionUtils.newHashMap(+ "name", containerName,+ "image", containerExecutable.getImage());+ if (containerExecutable.isUseTTY())+ stepContainerSpec.put("tty", true);+ List<Object> volumeMounts = new ArrayList<>(commonVolumeMounts);+ if (containerExecutable.getWorkingDir() != null) {+ volumeMounts.add(CollectionUtils.newLinkedHashMap(+ "name", "build-home",+ "mountPath", containerExecutable.getWorkingDir(),+ "subPath", "workspace"));+ }+ stepContainerSpec.put("volumeMounts", volumeMounts);+ List<Map<Object, Object>> envs = new ArrayList<>(commonEnvs);+ for (Map.Entry<String, String> entry: containerExecutable.getEnvMap().entrySet()) {+ envs.add(CollectionUtils.newLinkedHashMap(+ "name", entry.getKey(),+ "value", entry.getValue()));+ }+ stepContainerSpec.put("env", envs);} else {stepContainerSpec = CollectionUtils.newHashMap("name", containerName,"image", "1dev/k8s-helper-" + baselineOsInfo.getHelperImageSuffix() + ":" + helperImageVersion);+ stepContainerSpec.put("volumeMounts", commonVolumeMounts);+ stepContainerSpec.put("env", commonEnvs);}String positionStr = stringifyPosition(position);@@ -811,8 +846,6 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestDatastepContainerSpec.put("args", Lists.newArrayList("/c", containerCommandHome + "\\" + positionStr + ".bat"));}- stepContainerSpec.put("env", envs);- stepContainerSpec.put("volumeMounts", volumeMounts);containerSpecs.add(stepContainerSpec);return null;@@ -843,16 +876,16 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData"image", "1dev/k8s-helper-" + baselineOsInfo.getHelperImageSuffix() + ":" + helperImageVersion,"command", Lists.newArrayList("java"),"args", initArgs,- "env", envs,- "volumeMounts", volumeMounts);+ "env", commonEnvs,+ "volumeMounts", commonVolumeMounts);Map<Object, Object> sidecarContainerSpec = CollectionUtils.newHashMap("name", "sidecar","image", "1dev/k8s-helper-" + baselineOsInfo.getHelperImageSuffix() + ":" + helperImageVersion,"command", Lists.newArrayList("java"),"args", sidecarArgs,- "env", envs,- "volumeMounts", volumeMounts);+ "env", commonEnvs,+ "volumeMounts", commonVolumeMounts);if (jobContext != null) {sidecarContainerSpec.put("resources", CollectionUtils.newLinkedHashMap("requests", CollectionUtils.newLinkedHashMap(@@ -886,18 +919,22 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData"type", "DirectoryOrCreate"));List<Object> volumes = Lists.<Object>newArrayList(buildHomeVolume, userHomeVolume, cacheHomeVolume);if (trustCertsConfigMapName != null) {- Map<Object, Object> trustCertsHomeVolume = CollectionUtils.newLinkedHashMap(+ volumes.add(CollectionUtils.newLinkedHashMap("name", "trust-certs-home","configMap", CollectionUtils.newLinkedHashMap(- "name", trustCertsConfigMapName));- volumes.add(trustCertsHomeVolume);+ "name", trustCertsConfigMapName)));}if (dockerSock != null) {- Map<Object, Object> dockerSockVolume = CollectionUtils.newLinkedHashMap(+ volumes.add(CollectionUtils.newLinkedHashMap("name", "docker-sock","hostPath", CollectionUtils.newLinkedHashMap(- "path", dockerSock));- volumes.add(dockerSockVolume);+ "path", dockerSock)));+ }+ if (containerdSock != null) {+ volumes.add(CollectionUtils.newLinkedHashMap(+ "name", "containerd-sock",+ "hostPath", CollectionUtils.newLinkedHashMap(+ "path", containerdSock)));}podSpec.put("volumes", volumes);diff --git a/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java b/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.javaindex 42613dd2c0..f404aadfa0 100644--- a/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java+++ b/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java@@ -19,6 +19,7 @@ import java.io.File;import java.io.IOException;import java.io.Serializable;import java.util.ArrayList;+import java.util.Arrays;import java.util.Collection;import java.util.Date;import java.util.HashMap;@@ -31,6 +32,7 @@ import java.util.concurrent.atomic.AtomicBoolean;import java.util.concurrent.atomic.AtomicReference;import java.util.function.Consumer;+import javax.annotation.Nullable;import javax.validation.ConstraintValidatorContext;import org.apache.commons.lang3.SystemUtils;@@ -59,6 +61,7 @@ import io.onedev.k8shelper.CheckoutExecutable;import io.onedev.k8shelper.CloneInfo;import io.onedev.k8shelper.CommandExecutable;import io.onedev.k8shelper.CompositeExecutable;+import io.onedev.k8shelper.ContainerExecutable;import io.onedev.k8shelper.KubernetesHelper;import io.onedev.k8shelper.LeafExecutable;import io.onedev.k8shelper.LeafHandler;@@ -188,21 +191,8 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDastartService(newDocker(), network, jobService.toMap(), jobLogger);}- AtomicReference<File> workspaceCache = new AtomicReference<>(null);- for (Map.Entry<CacheInstance, String> entry: cacheAllocations.entrySet()) {- if (PathUtils.isCurrent(entry.getValue())) {- workspaceCache.set(entry.getKey().getDirectory(hostCacheHome));- break;- }- }-- File hostWorkspace;- if (workspaceCache.get() != null) {- hostWorkspace = workspaceCache.get();- } else {- hostWorkspace = new File(hostBuildHome, "workspace");- FileUtils.createDir(hostWorkspace);- }+ File hostWorkspace = new File(hostBuildHome, "workspace");+ FileUtils.createDir(hostWorkspace);AtomicReference<File> hostAuthInfoHome = new AtomicReference<>(null);try {@@ -224,6 +214,61 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDaboolean successful = entryExecutable.execute(new LeafHandler() {+ private int runStepContainer(String image, @Nullable String entrypoint,+ List<String> arguments, Map<String, String> environments,+ @Nullable String workingDir, List<Integer> position, boolean useTTY) {+ String containerName = network + "-step-" + stringifyPosition(position);+ Commandline docker = newDocker();+ docker.addArgs("run", "--name=" + containerName, "--network=" + network);+ if (getRunOptions() != null)+ docker.addArgs(StringUtils.parseQuoteTokens(getRunOptions()));++ docker.addArgs("-v", getOuterPath(hostBuildHome.getAbsolutePath()) + ":" + containerBuildHome);+ if (workingDir != null) {+ docker.addArgs("-v", getOuterPath(hostWorkspace.getAbsolutePath()) + ":" + workingDir);+ docker.addArgs("-w", workingDir);+ } else {+ docker.addArgs("-w", containerWorkspace);+ }+ for (Map.Entry<CacheInstance, String> entry: cacheAllocations.entrySet()) {+ if (!PathUtils.isCurrent(entry.getValue())) {+ String each = entry.getKey().getDirectory(hostCacheHome).getAbsolutePath();+ String containerCachePath = PathUtils.resolve(containerWorkspace, entry.getValue());+ docker.addArgs("-v", getOuterPath(each) + ":" + containerCachePath);+ } else {+ throw new ExplicitException("Invalid cache path: " + entry.getValue());+ }+ }++ if (SystemUtils.IS_OS_LINUX)+ docker.addArgs("-v", "/var/run/docker.sock:/var/run/docker.sock");++ if (hostAuthInfoHome.get() != null) {+ String outerPath = getOuterPath(hostAuthInfoHome.get().getAbsolutePath());+ if (SystemUtils.IS_OS_WINDOWS) {+ docker.addArgs("-v", outerPath + ":C:\\Users\\ContainerAdministrator\\auth-info");+ docker.addArgs("-v", outerPath + ":C:\\Users\\ContainerUser\\auth-info");+ } else {+ docker.addArgs("-v", outerPath + ":/root/auth-info");+ }+ }++ for (Map.Entry<String, String> entry: environments.entrySet())+ docker.addArgs("-e", entry.getKey() + "=" + entry.getValue());++ if (useTTY)+ docker.addArgs("-t");++ if (entrypoint != null)+ docker.addArgs("--entrypoint=" + entrypoint);++ docker.addArgs(image);+ docker.addArgs(arguments.toArray(new String[arguments.size()]));++ ProcessKiller killer = newDockerKiller(newDocker(), containerName, jobLogger);+ return docker.execute(newInfoLogger(jobLogger), newErrorLogger(jobLogger), null, killer).getReturnCode();+ }+@Overridepublic boolean execute(LeafExecutable executable, List<Integer> position) {String stepNames = entryExecutable.getNamesAsString(position);@@ -233,55 +278,31 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDaCommandExecutable commandExecutable = (CommandExecutable) executable;if (commandExecutable.getImage() == null) {- throw new ExplicitException("This step should be executed by server shell "+ throw new ExplicitException("This step can only be executed by server shell "+ "executor or remote shell executor");}Commandline entrypoint = DockerExecutorUtils.getEntrypoint(hostBuildHome, commandExecutable, hostAuthInfoHome.get() != null);- String containerName = network + "-step-" + stringifyPosition(position);- Commandline docker = newDocker();- docker.clearArgs();- docker.addArgs("run", "--name=" + containerName, "--network=" + network);- if (getRunOptions() != null)- docker.addArgs(StringUtils.parseQuoteTokens(getRunOptions()));-- docker.addArgs("-v", getOuterPath(hostBuildHome.getAbsolutePath()) + ":" + containerBuildHome);- if (workspaceCache.get() != null)- docker.addArgs("-v", getOuterPath(workspaceCache.get().getAbsolutePath()) + ":" + containerWorkspace);- for (Map.Entry<CacheInstance, String> entry: cacheAllocations.entrySet()) {- if (!PathUtils.isCurrent(entry.getValue())) {- String each = entry.getKey().getDirectory(hostCacheHome).getAbsolutePath();- String containerCachePath = PathUtils.resolve(containerWorkspace, entry.getValue());- docker.addArgs("-v", getOuterPath(each) + ":" + containerCachePath);- }- }-- if (SystemUtils.IS_OS_LINUX)- docker.addArgs("-v", "/var/run/docker.sock:/var/run/docker.sock");+ int exitCode = runStepContainer(commandExecutable.getImage(), entrypoint.executable(),+ entrypoint.arguments(), new HashMap<>(), null, position, commandExecutable.isUseTTY());- if (hostAuthInfoHome.get() != null) {- String outerPath = getOuterPath(hostAuthInfoHome.get().getAbsolutePath());- if (SystemUtils.IS_OS_WINDOWS) {- docker.addArgs("-v", outerPath + ":C:\\Users\\ContainerAdministrator\\auth-info");- docker.addArgs("-v", outerPath + ":C:\\Users\\ContainerUser\\auth-info");- } else {- docker.addArgs("-v", outerPath + ":/root/auth-info");- }+ if (exitCode != 0) {+ jobLogger.error("Step \"" + stepNames + "\" is failed: Command exited with code " + exitCode);+ return false;}-- if (commandExecutable.isUseTTY())- docker.addArgs("-t");- docker.addArgs("-w", containerWorkspace, "--entrypoint=" + entrypoint.executable());-- docker.addArgs(commandExecutable.getImage());- docker.addArgs(entrypoint.arguments().toArray(new String[0]));-- ProcessKiller killer = newDockerKiller(newDocker(), containerName, jobLogger);- ExecutionResult result = docker.execute(newInfoLogger(jobLogger), newErrorLogger(jobLogger), null, killer);- if (result.getReturnCode() != 0) {- jobLogger.error("Step \"" + stepNames + "\" is failed: Command failed with exit code " + result.getReturnCode());+ } else if (executable instanceof ContainerExecutable) {+ ContainerExecutable containerExecutable = (ContainerExecutable) executable;++ List<String> arguments = new ArrayList<>();+ if (containerExecutable.getArgs() != null)+ arguments.addAll(Arrays.asList(StringUtils.parseQuoteTokens(containerExecutable.getArgs())));+ int exitCode = runStepContainer(containerExecutable.getImage(), null, arguments,+ containerExecutable.getEnvMap(), containerExecutable.getWorkingDir(),+ position, containerExecutable.isUseTTY());+ if (exitCode != 0) {+ jobLogger.error("Step \"" + stepNames + "\" is failed: Container exited with code " + exitCode);return false;}} else if (executable instanceof CheckoutExecutable) {diff --git a/server-plugin/server-plugin-executor-servershell/src/main/java/io/onedev/server/plugin/executor/servershell/ServerShellExecutor.java b/server-plugin/server-plugin-executor-servershell/src/main/java/io/onedev/server/plugin/executor/servershell/ServerShellExecutor.javaindex 8074432e12..cc81425809 100644--- a/server-plugin/server-plugin-executor-servershell/src/main/java/io/onedev/server/plugin/executor/servershell/ServerShellExecutor.java+++ b/server-plugin/server-plugin-executor-servershell/src/main/java/io/onedev/server/plugin/executor/servershell/ServerShellExecutor.java@@ -40,6 +40,7 @@ import io.onedev.k8shelper.CheckoutExecutable;import io.onedev.k8shelper.CloneInfo;import io.onedev.k8shelper.CommandExecutable;import io.onedev.k8shelper.CompositeExecutable;+import io.onedev.k8shelper.ContainerExecutable;import io.onedev.k8shelper.LeafExecutable;import io.onedev.k8shelper.LeafHandler;import io.onedev.k8shelper.ServerExecutable;@@ -114,21 +115,8 @@ public class ServerShellExecutor extends JobExecutor implements Testable<TestDat});- File workspaceCache = null;- for (Map.Entry<CacheInstance, String> entry: cacheAllocations.entrySet()) {- if (PathUtils.isCurrent(entry.getValue())) {- workspaceCache = entry.getKey().getDirectory(cacheHomeDir);- break;- }- }-- File workspaceDir;- if (workspaceCache != null) {- workspaceDir = workspaceCache;- } else {- workspaceDir = new File(buildDir, "workspace");- FileUtils.createDir(workspaceDir);- }+ File workspaceDir = new File(buildDir, "workspace");+ FileUtils.createDir(workspaceDir);jobLogger.log("Copying job dependencies...");jobContext.copyDependencies(workspaceDir);@@ -150,7 +138,7 @@ public class ServerShellExecutor extends JobExecutor implements Testable<TestDatif (executable instanceof CommandExecutable) {CommandExecutable commandExecutable = (CommandExecutable) executable;if (commandExecutable.getImage() != null) {- throw new ExplicitException("This step should be executed by server docker executor, "+ throw new ExplicitException("This step can only be executed by server docker executor, "+ "remote docker executor, or kubernetes executor");}@@ -177,6 +165,8 @@ public class ServerShellExecutor extends JobExecutor implements Testable<TestDat} catch (IOException e) {throw new RuntimeException(e);}+ } else {+ throw new ExplicitException("Invalid cache path: " + entry.getValue());}}@@ -188,9 +178,12 @@ public class ServerShellExecutor extends JobExecutor implements Testable<TestDatExecutionResult result = interpreter.execute(newInfoLogger(jobLogger), newErrorLogger(jobLogger));if (result.getReturnCode() != 0) {- jobLogger.error("Step \"" + stepNames + "\" is failed: Command failed with exit code " + result.getReturnCode());+ jobLogger.error("Step \"" + stepNames + "\" is failed: Command exited with code " + result.getReturnCode());return false;}+ } else if (executable instanceof ContainerExecutable) {+ throw new ExplicitException("This step can only be executed by server docker executor, "+ + "remote docker executor, or kubernetes executor");} else if (executable instanceof CheckoutExecutable) {try {CheckoutExecutable checkoutExecutable = (CheckoutExecutable) executable;diff --git a/server-plugin/server-plugin-report-checkstyle/src/main/java/io/onedev/server/plugin/report/checkstyle/PublishESLintReportStep.java b/server-plugin/server-plugin-report-checkstyle/src/main/java/io/onedev/server/plugin/report/checkstyle/PublishESLintReportStep.javaindex b8c4a1470a..96d5a3bb2d 100644--- a/server-plugin/server-plugin-report-checkstyle/src/main/java/io/onedev/server/plugin/report/checkstyle/PublishESLintReportStep.java+++ b/server-plugin/server-plugin-report-checkstyle/src/main/java/io/onedev/server/plugin/report/checkstyle/PublishESLintReportStep.java@@ -6,7 +6,7 @@ import io.onedev.server.web.editable.annotation.Editable;import io.onedev.server.web.editable.annotation.Interpolative;import io.onedev.server.web.editable.annotation.Patterns;-@Editable(order=300, name="Publish ESLint Report")+@Editable(order=3000, name="Publish ESLint Report")public class PublishESLintReportStep extends PublishCheckstyleReportStep {private static final long serialVersionUID = 1L;diff --git a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PublishMarkdownReportStep.java b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PublishMarkdownReportStep.javaindex 08c1917ce7..39853577be 100644--- a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PublishMarkdownReportStep.java+++ b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PublishMarkdownReportStep.java@@ -18,7 +18,7 @@ import io.onedev.server.model.Build;import io.onedev.server.web.editable.annotation.Editable;import io.onedev.server.web.editable.annotation.Interpolative;-@Editable(order=110, name="Publish Markdown Report")+@Editable(order=1100, name="Publish Markdown Report")public class PublishMarkdownReportStep extends PublishReportStep {private static final long serialVersionUID = 1L;diff --git a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PublishPullRequestMarkdownReportStep.java b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PublishPullRequestMarkdownReportStep.javaindex 4e03164913..485472f53a 100644--- a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PublishPullRequestMarkdownReportStep.java+++ b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PublishPullRequestMarkdownReportStep.java@@ -18,7 +18,7 @@ import io.onedev.server.model.Build;import io.onedev.server.web.editable.annotation.Editable;import io.onedev.server.web.editable.annotation.Interpolative;-@Editable(order=120, name="Publish Markdown Report (for Pull Request)", description="This report will be displayed in pull request overview "+@Editable(order=1200, name="Publish Markdown Report (for Pull Request)", description="This report will be displayed in pull request overview "+ "page if the build belongs to a pull request")public class PublishPullRequestMarkdownReportStep extends PublishReportStep {diff --git a/server-product/docker/Dockerfile b/server-product/docker/Dockerfileindex 30c6e42ebb..525e71da70 100644--- a/server-product/docker/Dockerfile+++ b/server-product/docker/Dockerfile@@ -3,7 +3,8 @@ FROM ubuntu:18.04COPY app /appCOPY entrypoint.sh idle.sh /root/bin/COPY kubectl /usr/local/bin/-RUN apt-get update && apt-get install -y locales docker.io iproute2 curl git git-lfs openjdk-8-jre-headless+COPY docker /usr/local/bin/+RUN apt-get update && apt-get install -y locales iproute2 curl git git-lfs openjdk-8-jre-headless && rm -rf /var/lib/apt/lists/*RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \locale-genENV LANG en_US.UTF-8diff --git a/server-product/docker/docker b/server-product/docker/dockernew file mode 100755index 0000000000..ee72d7b096Binary files /dev/null and b/server-product/docker/docker differ