Skip to content

The Unified and Autonomous Development Platform

Run container step support for linux

Robin Shenba5575aa8266
diff --git a/doc/incompatibilities.md b/doc/incompatibilities.md
index 5b386032e3..a16030df1b 100644
--- a/doc/incompatibilities.md
+++ b/doc/incompatibilities.md
@@ -2,6 +2,7 @@
1. A new property `Run In Container` is added and enabled for all command steps in build spec. If you have steps intended to
be executed by shell executor, edit it to disable this property, otherwise the build will fail reporting `This step should be executed by server docker executor, remote docker executor, or kubernetes executor`
+1. Using cache path "." (to cache workspace itself) in job cache definition is no longer supported
# 5.4.0
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/job/CacheSpec.java b/server-core/src/main/java/io/onedev/server/buildspec/job/CacheSpec.java
index 3fe1352b76..19108e75f5 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/job/CacheSpec.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/job/CacheSpec.java
@@ -3,16 +3,22 @@ package io.onedev.server.buildspec.job;
import java.io.Serializable;
import java.util.List;
+import javax.validation.ConstraintValidatorContext;
+
import org.hibernate.validator.constraints.NotEmpty;
import io.onedev.commons.codeassist.InputSuggestion;
+import io.onedev.commons.utils.PathUtils;
import io.onedev.server.buildspec.BuildSpec;
+import io.onedev.server.util.validation.Validatable;
+import io.onedev.server.util.validation.annotation.ClassValidating;
import io.onedev.server.util.validation.annotation.RegEx;
import io.onedev.server.web.editable.annotation.Editable;
import io.onedev.server.web.editable.annotation.Interpolative;
@Editable
-public class CacheSpec implements Serializable {
+@ClassValidating
+public class CacheSpec implements Serializable, Validatable {
private static final long serialVersionUID = 1L;
@@ -20,7 +26,8 @@ public class CacheSpec implements Serializable {
private String path;
- @Editable(order=100, description="Specify key of the cache. Caches with same key can be reused by different jobs")
+ @Editable(order=100, description="Specify key of the cache. Caches with same key can be reused by different projects/jobs. "
+ + "Embed project/job variable to prevent cross project/job reuse")
@Interpolative(variableSuggester="suggestVariables")
@NotEmpty
@RegEx(pattern="[a-zA-Z0-9\\-_\\.]+", message="Can only contain alphanumeric, dash, dot and underscore")
@@ -33,8 +40,7 @@ public class CacheSpec implements Serializable {
}
@Editable(order=200, description="Specify path to cache. Non-absolute path is considered to be relative to job workspace. "
- + "Specify \".\" (without quote) to cache workspace itself. "
- + "<span class='text-warning'>Absolute path is not allowed if the job is executed by a shell executor</span>")
+ + "Please note that shell executor only allows non-absolute path here")
@Interpolative(variableSuggester="suggestVariables")
@NotEmpty
public String getPath() {
@@ -50,4 +56,15 @@ public class CacheSpec implements Serializable {
return BuildSpec.suggestVariables(matchWith, false, false);
}
+ @Override
+ public boolean isValid(ConstraintValidatorContext context) {
+ if (PathUtils.isCurrent(path)) {
+ context.disableDefaultConstraintViolation();
+ context.buildConstraintViolationWithTemplate("Invalid path").addPropertyNode("path").addConstraintViolation();
+ return false;
+ } else {
+ return true;
+ }
+ }
+
}
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/CheckoutStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/CheckoutStep.java
index 6baf9f959e..148bb2007a 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/step/CheckoutStep.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/CheckoutStep.java
@@ -10,7 +10,7 @@ import io.onedev.server.buildspec.param.ParamCombination;
import io.onedev.server.model.Build;
import io.onedev.server.web.editable.annotation.Editable;
-@Editable(order=5, name="Checkout Code")
+@Editable(order=50, name="Checkout Code")
public class CheckoutStep extends Step {
private static final long serialVersionUID = 1L;
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/CloseMilestoneStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/CloseMilestoneStep.java
index dcd9ba96ae..b3df2e0da7 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/step/CloseMilestoneStep.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/CloseMilestoneStep.java
@@ -19,7 +19,7 @@ import io.onedev.server.persistence.TransactionManager;
import io.onedev.server.web.editable.annotation.Editable;
import io.onedev.server.web.editable.annotation.Interpolative;
-@Editable(name="Close Milestone", order=40)
+@Editable(name="Close Milestone", order=400)
public class CloseMilestoneStep extends ServerStep {
private static final long serialVersionUID = 1L;
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/CreateTagStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/CreateTagStep.java
index cf5dfa8b0c..1d4498daa5 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/step/CreateTagStep.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/CreateTagStep.java
@@ -21,7 +21,7 @@ import io.onedev.server.web.editable.annotation.Editable;
import io.onedev.server.web.editable.annotation.Interpolative;
import io.onedev.server.web.editable.annotation.Multiline;
-@Editable(name="Create Tag", order=30)
+@Editable(name="Create Tag", order=300)
public class CreateTagStep extends ServerStep {
private static final long serialVersionUID = 1L;
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/PublishArtifactStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/PublishArtifactStep.java
index 6ed3dd139c..5417de8ddc 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/step/PublishArtifactStep.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/PublishArtifactStep.java
@@ -18,7 +18,7 @@ import io.onedev.server.web.editable.annotation.Editable;
import io.onedev.server.web.editable.annotation.Interpolative;
import io.onedev.server.web.editable.annotation.Patterns;
-@Editable(order=105, name="Publish Artifacts")
+@Editable(order=1050, name="Publish Artifacts")
public class PublishArtifactStep extends ServerStep {
private static final long serialVersionUID = 1L;
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/RunContainerStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/RunContainerStep.java
new file mode 100644
index 0000000000..080f2b06be
--- /dev/null
+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/RunContainerStep.java
@@ -0,0 +1,108 @@
+package io.onedev.server.buildspec.step;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+import org.hibernate.validator.constraints.NotEmpty;
+
+import io.onedev.commons.codeassist.InputSuggestion;
+import io.onedev.k8shelper.ContainerExecutable;
+import io.onedev.k8shelper.Executable;
+import io.onedev.server.buildspec.BuildSpec;
+import io.onedev.server.buildspec.job.EnvVar;
+import io.onedev.server.buildspec.param.ParamCombination;
+import io.onedev.server.model.Build;
+import io.onedev.server.web.editable.annotation.Editable;
+import io.onedev.server.web.editable.annotation.Interpolative;
+
+@Editable(order=150, name="Run Container")
+public class RunContainerStep extends Step {
+
+ private static final long serialVersionUID = 1L;
+
+ private String image;
+
+ private String args;
+
+ private List<EnvVar> envVars = new ArrayList<>();
+
+ private String workingDir;
+
+ private boolean useTTY;
+
+ @Override
+ public Executable getExecutable(Build build, String jobToken, ParamCombination paramCombination) {
+ Map<String, String> envMap = new HashMap<>();
+ for (EnvVar var: getEnvVars())
+ envMap.put(var.getName(), var.getValue());
+ return new ContainerExecutable(getImage(), getArgs(), envMap, getWorkingDir(), isUseTTY());
+ }
+
+ @Editable(order=100, description="Specify container image to run. <b class='text-warning'>NOTE:</b> A shell must "
+ + "exist in the container if the step is executed by kubernetes executor, as OneDev intercepts the "
+ + "entrypoint to make step containers executing sequentially in the pod")
+ @Interpolative(variableSuggester="suggestVariables")
+ @NotEmpty
+ public String getImage() {
+ return image;
+ }
+
+ public void setImage(String image) {
+ this.image = image;
+ }
+
+ @Editable(order=200, name="Arguments", description="Specify container arguments separated by space. "
+ + "Single argument containing space should be quoted")
+ @Interpolative(variableSuggester="suggestVariables")
+ public String getArgs() {
+ return args;
+ }
+
+ public void setArgs(String args) {
+ this.args = args;
+ }
+
+ @Editable(order=200, name="Working Directory", description="Optionally specify working directory of the container. "
+ + "Leave empty to use the default working directory (<tt>/onedev-build/workspace</tt>). "
+ + "<a href='$docRoot/pages/concepts.md#job-workspace' target='_blank'>Job workspace</a> will be "
+ + "mounted to the working directory so that the container can access job files")
+ @Nullable
+ public String getWorkingDir() {
+ return workingDir;
+ }
+
+ public void setWorkingDir(String workingDir) {
+ this.workingDir = workingDir;
+ }
+
+ @Editable(order=400, name="Environment Variables", description="Optionally specify environment "
+ + "variables for the container")
+ public List<EnvVar> getEnvVars() {
+ return envVars;
+ }
+
+ public void setEnvVars(List<EnvVar> envVars) {
+ this.envVars = envVars;
+ }
+
+ @Editable(order=10000, name="Enable TTY Mode", description="Many commands print outputs with ANSI colors in "
+ + "TTY mode to help identifying problems easily. However some commands running in this mode may "
+ + "wait for user input to cause build hanging. This can normally be fixed by adding extra options "
+ + "to the command")
+ public boolean isUseTTY() {
+ return useTTY;
+ }
+
+ public void setUseTTY(boolean useTTY) {
+ this.useTTY = useTTY;
+ }
+
+ static List<InputSuggestion> suggestVariables(String matchWith) {
+ return BuildSpec.suggestVariables(matchWith, false, false);
+ }
+
+}
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/SetBuildVersionStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/SetBuildVersionStep.java
index 3f24fab0c7..7bd96b2d0f 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/step/SetBuildVersionStep.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/SetBuildVersionStep.java
@@ -22,7 +22,7 @@ import io.onedev.server.persistence.TransactionManager;
import io.onedev.server.web.editable.annotation.Editable;
import io.onedev.server.web.editable.annotation.Interpolative;
-@Editable(order=20, name="Set Build Version")
+@Editable(order=260, name="Set Build Version")
public class SetBuildVersionStep extends ServerStep {
private static final long serialVersionUID = 1L;
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/command/CommandStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/command/CommandStep.java
index 922f0d99bd..b7f24e81d6 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/step/command/CommandStep.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/command/CommandStep.java
@@ -19,7 +19,7 @@ import io.onedev.server.web.editable.annotation.Editable;
import io.onedev.server.web.editable.annotation.Interpolative;
import io.onedev.server.web.editable.annotation.ShowCondition;
-@Editable(order=10, name="Execute Commands")
+@Editable(order=100, name="Execute Commands")
public class CommandStep extends Step {
private static final long serialVersionUID = 1L;
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/command/interpreter/BorneShellInterpreter.java b/server-core/src/main/java/io/onedev/server/buildspec/step/command/interpreter/BashInterpreter.java
similarity index 75%
rename from server-core/src/main/java/io/onedev/server/buildspec/step/command/interpreter/BorneShellInterpreter.java
rename to server-core/src/main/java/io/onedev/server/buildspec/step/command/interpreter/BashInterpreter.java
index 2ef75a7ef2..a2808f4f1e 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/step/command/interpreter/BorneShellInterpreter.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/command/interpreter/BashInterpreter.java
@@ -4,18 +4,18 @@ import java.util.List;
import javax.validation.constraints.Size;
-import io.onedev.k8shelper.BorneShellExecutable;
+import io.onedev.k8shelper.BashExecutable;
import io.onedev.k8shelper.CommandExecutable;
import io.onedev.server.util.validation.annotation.Code;
import io.onedev.server.web.editable.annotation.Editable;
import io.onedev.server.web.editable.annotation.Interpolative;
-@Editable(order=200, name="Borne Shell")
-public class BorneShellInterpreter extends Interpreter {
+@Editable(order=200, name="bash")
+public class BashInterpreter extends Interpreter {
private static final long serialVersionUID = 1L;
- @Editable(order=110, description="Specify borne shell commands to execute "
+ @Editable(order=110, description="Specify bash commands to execute "
+ "under the <a href='$docRoot/pages/concepts.md#job-workspace' target='_blank'>job workspace</a>")
@Interpolative
@Code(language=Code.SHELL, variableProvider="suggestVariables")
@@ -30,7 +30,7 @@ public class BorneShellInterpreter extends Interpreter {
@Override
public CommandExecutable getExecutable(String image, boolean useTTY) {
- return new BorneShellExecutable(image, getCommands(), useTTY);
+ return new BashExecutable(image, getCommands(), useTTY);
}
}
diff --git a/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesExecutor.java b/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesExecutor.java
index 77a193b926..2da094a22d 100644
--- a/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesExecutor.java
+++ b/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesExecutor.java
@@ -56,6 +56,7 @@ import io.onedev.commons.utils.command.LineConsumer;
import io.onedev.k8shelper.Action;
import io.onedev.k8shelper.CommandExecutable;
import io.onedev.k8shelper.CompositeExecutable;
+import io.onedev.k8shelper.ContainerExecutable;
import io.onedev.k8shelper.ExecuteCondition;
import io.onedev.k8shelper.KubernetesHelper;
import io.onedev.k8shelper.LeafExecutable;
@@ -701,6 +702,7 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
String containerAuthInfoHome;
String trustCertsHome;
String dockerSock;
+ String containerdSock;
if (baselineOsInfo.isLinux()) {
containerBuildHome = "/onedev-build";
containerCacheHome = containerBuildHome + "/cache";
@@ -708,6 +710,7 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
containerAuthInfoHome = "/root/auth-info";
trustCertsHome = containerBuildHome + "/trust-certs";
dockerSock = "/var/run/docker.sock";
+ containerdSock = "/run/containerd/containerd.sock";
} else {
containerBuildHome = "C:\\onedev-build";
containerCacheHome = containerBuildHome + "\\cache";
@@ -715,6 +718,7 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
containerAuthInfoHome = "C:\\Users\\ContainerAdministrator\\auth-info";
trustCertsHome = containerBuildHome + "\\trust-certs";
dockerSock = null;
+ containerdSock = null;
}
Map<String, String> buildHomeMount = CollectionUtils.newLinkedHashMap(
@@ -738,14 +742,19 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
Map<String, String> dockerSockMount = CollectionUtils.newLinkedHashMap(
"name", "docker-sock",
"mountPath", dockerSock);
+ Map<String, String> containerdSockMount = CollectionUtils.newLinkedHashMap(
+ "name", "containerd-sock",
+ "mountPath", containerdSock);
- List<Object> volumeMounts = Lists.<Object>newArrayList(buildHomeMount, authInfoMount, cacheHomeMount);
+ List<Object> commonVolumeMounts = Lists.<Object>newArrayList(buildHomeMount, authInfoMount, cacheHomeMount);
if (baselineOsInfo.isWindows())
- volumeMounts.add(authInfoMount2);
+ commonVolumeMounts.add(authInfoMount2);
if (trustCertsConfigMapName != null)
- volumeMounts.add(trustCertsMount);
+ commonVolumeMounts.add(trustCertsMount);
if (dockerSock != null)
- volumeMounts.add(dockerSockMount);
+ commonVolumeMounts.add(dockerSockMount);
+ if (containerdSock != null)
+ commonVolumeMounts.add(containerdSockMount);
CompositeExecutable entryExecutable;
if (jobContext != null) {
@@ -769,11 +778,11 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
throw new RuntimeException(e);
}
- List<Map<Object, Object>> envs = new ArrayList<>();
- envs.add(CollectionUtils.newLinkedHashMap(
+ List<Map<Object, Object>> commonEnvs = new ArrayList<>();
+ commonEnvs.add(CollectionUtils.newLinkedHashMap(
"name", ENV_SERVER_URL,
"value", getServerUrl()));
- envs.add(CollectionUtils.newLinkedHashMap(
+ commonEnvs.add(CollectionUtils.newLinkedHashMap(
"name", ENV_JOB_TOKEN,
"value", jobToken));
@@ -787,19 +796,45 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
if (executable instanceof CommandExecutable) {
CommandExecutable commandExecutable = (CommandExecutable) executable;
if (commandExecutable.getImage() == null) {
- throw new ExplicitException("This step should be executed by server shell "
+ throw new ExplicitException("This step can only be executed by server shell "
+ "executor or remote shell executor");
}
stepContainerSpec = CollectionUtils.newHashMap(
"name", containerName,
"image", commandExecutable.getImage());
- if (((CommandExecutable) executable).isUseTTY())
+ if (commandExecutable.isUseTTY())
stepContainerSpec.put("tty", true);
+ stepContainerSpec.put("volumeMounts", commonVolumeMounts);
+ stepContainerSpec.put("env", commonEnvs);
+ } else if (executable instanceof ContainerExecutable) {
+ ContainerExecutable containerExecutable = (ContainerExecutable) executable;
+ stepContainerSpec = CollectionUtils.newHashMap(
+ "name", containerName,
+ "image", containerExecutable.getImage());
+ if (containerExecutable.isUseTTY())
+ stepContainerSpec.put("tty", true);
+ List<Object> volumeMounts = new ArrayList<>(commonVolumeMounts);
+ if (containerExecutable.getWorkingDir() != null) {
+ volumeMounts.add(CollectionUtils.newLinkedHashMap(
+ "name", "build-home",
+ "mountPath", containerExecutable.getWorkingDir(),
+ "subPath", "workspace"));
+ }
+ stepContainerSpec.put("volumeMounts", volumeMounts);
+ List<Map<Object, Object>> envs = new ArrayList<>(commonEnvs);
+ for (Map.Entry<String, String> entry: containerExecutable.getEnvMap().entrySet()) {
+ envs.add(CollectionUtils.newLinkedHashMap(
+ "name", entry.getKey(),
+ "value", entry.getValue()));
+ }
+ stepContainerSpec.put("env", envs);
} else {
stepContainerSpec = CollectionUtils.newHashMap(
"name", containerName,
"image", "1dev/k8s-helper-" + baselineOsInfo.getHelperImageSuffix() + ":" + helperImageVersion);
+ stepContainerSpec.put("volumeMounts", commonVolumeMounts);
+ stepContainerSpec.put("env", commonEnvs);
}
String positionStr = stringifyPosition(position);
@@ -811,8 +846,6 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
stepContainerSpec.put("args", Lists.newArrayList("/c", containerCommandHome + "\\" + positionStr + ".bat"));
}
- stepContainerSpec.put("env", envs);
- stepContainerSpec.put("volumeMounts", volumeMounts);
containerSpecs.add(stepContainerSpec);
return null;
@@ -843,16 +876,16 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
"image", "1dev/k8s-helper-" + baselineOsInfo.getHelperImageSuffix() + ":" + helperImageVersion,
"command", Lists.newArrayList("java"),
"args", initArgs,
- "env", envs,
- "volumeMounts", volumeMounts);
+ "env", commonEnvs,
+ "volumeMounts", commonVolumeMounts);
Map<Object, Object> sidecarContainerSpec = CollectionUtils.newHashMap(
"name", "sidecar",
"image", "1dev/k8s-helper-" + baselineOsInfo.getHelperImageSuffix() + ":" + helperImageVersion,
"command", Lists.newArrayList("java"),
"args", sidecarArgs,
- "env", envs,
- "volumeMounts", volumeMounts);
+ "env", commonEnvs,
+ "volumeMounts", commonVolumeMounts);
if (jobContext != null) {
sidecarContainerSpec.put("resources", CollectionUtils.newLinkedHashMap("requests", CollectionUtils.newLinkedHashMap(
@@ -886,18 +919,22 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
"type", "DirectoryOrCreate"));
List<Object> volumes = Lists.<Object>newArrayList(buildHomeVolume, userHomeVolume, cacheHomeVolume);
if (trustCertsConfigMapName != null) {
- Map<Object, Object> trustCertsHomeVolume = CollectionUtils.newLinkedHashMap(
+ volumes.add(CollectionUtils.newLinkedHashMap(
"name", "trust-certs-home",
"configMap", CollectionUtils.newLinkedHashMap(
- "name", trustCertsConfigMapName));
- volumes.add(trustCertsHomeVolume);
+ "name", trustCertsConfigMapName)));
}
if (dockerSock != null) {
- Map<Object, Object> dockerSockVolume = CollectionUtils.newLinkedHashMap(
+ volumes.add(CollectionUtils.newLinkedHashMap(
"name", "docker-sock",
"hostPath", CollectionUtils.newLinkedHashMap(
- "path", dockerSock));
- volumes.add(dockerSockVolume);
+ "path", dockerSock)));
+ }
+ if (containerdSock != null) {
+ volumes.add(CollectionUtils.newLinkedHashMap(
+ "name", "containerd-sock",
+ "hostPath", CollectionUtils.newLinkedHashMap(
+ "path", containerdSock)));
}
podSpec.put("volumes", volumes);
diff --git a/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java b/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java
index 42613dd2c0..f404aadfa0 100644
--- a/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java
+++ b/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java
@@ -19,6 +19,7 @@ import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
@@ -31,6 +32,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
+import javax.annotation.Nullable;
import javax.validation.ConstraintValidatorContext;
import org.apache.commons.lang3.SystemUtils;
@@ -59,6 +61,7 @@ import io.onedev.k8shelper.CheckoutExecutable;
import io.onedev.k8shelper.CloneInfo;
import io.onedev.k8shelper.CommandExecutable;
import io.onedev.k8shelper.CompositeExecutable;
+import io.onedev.k8shelper.ContainerExecutable;
import io.onedev.k8shelper.KubernetesHelper;
import io.onedev.k8shelper.LeafExecutable;
import io.onedev.k8shelper.LeafHandler;
@@ -188,21 +191,8 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
startService(newDocker(), network, jobService.toMap(), jobLogger);
}
- AtomicReference<File> workspaceCache = new AtomicReference<>(null);
- for (Map.Entry<CacheInstance, String> entry: cacheAllocations.entrySet()) {
- if (PathUtils.isCurrent(entry.getValue())) {
- workspaceCache.set(entry.getKey().getDirectory(hostCacheHome));
- break;
- }
- }
-
- File hostWorkspace;
- if (workspaceCache.get() != null) {
- hostWorkspace = workspaceCache.get();
- } else {
- hostWorkspace = new File(hostBuildHome, "workspace");
- FileUtils.createDir(hostWorkspace);
- }
+ File hostWorkspace = new File(hostBuildHome, "workspace");
+ FileUtils.createDir(hostWorkspace);
AtomicReference<File> hostAuthInfoHome = new AtomicReference<>(null);
try {
@@ -224,6 +214,61 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
boolean successful = entryExecutable.execute(new LeafHandler() {
+ private int runStepContainer(String image, @Nullable String entrypoint,
+ List<String> arguments, Map<String, String> environments,
+ @Nullable String workingDir, List<Integer> position, boolean useTTY) {
+ String containerName = network + "-step-" + stringifyPosition(position);
+ Commandline docker = newDocker();
+ docker.addArgs("run", "--name=" + containerName, "--network=" + network);
+ if (getRunOptions() != null)
+ docker.addArgs(StringUtils.parseQuoteTokens(getRunOptions()));
+
+ docker.addArgs("-v", getOuterPath(hostBuildHome.getAbsolutePath()) + ":" + containerBuildHome);
+ if (workingDir != null) {
+ docker.addArgs("-v", getOuterPath(hostWorkspace.getAbsolutePath()) + ":" + workingDir);
+ docker.addArgs("-w", workingDir);
+ } else {
+ docker.addArgs("-w", containerWorkspace);
+ }
+ for (Map.Entry<CacheInstance, String> entry: cacheAllocations.entrySet()) {
+ if (!PathUtils.isCurrent(entry.getValue())) {
+ String each = entry.getKey().getDirectory(hostCacheHome).getAbsolutePath();
+ String containerCachePath = PathUtils.resolve(containerWorkspace, entry.getValue());
+ docker.addArgs("-v", getOuterPath(each) + ":" + containerCachePath);
+ } else {
+ throw new ExplicitException("Invalid cache path: " + entry.getValue());
+ }
+ }
+
+ if (SystemUtils.IS_OS_LINUX)
+ docker.addArgs("-v", "/var/run/docker.sock:/var/run/docker.sock");
+
+ if (hostAuthInfoHome.get() != null) {
+ String outerPath = getOuterPath(hostAuthInfoHome.get().getAbsolutePath());
+ if (SystemUtils.IS_OS_WINDOWS) {
+ docker.addArgs("-v", outerPath + ":C:\\Users\\ContainerAdministrator\\auth-info");
+ docker.addArgs("-v", outerPath + ":C:\\Users\\ContainerUser\\auth-info");
+ } else {
+ docker.addArgs("-v", outerPath + ":/root/auth-info");
+ }
+ }
+
+ for (Map.Entry<String, String> entry: environments.entrySet())
+ docker.addArgs("-e", entry.getKey() + "=" + entry.getValue());
+
+ if (useTTY)
+ docker.addArgs("-t");
+
+ if (entrypoint != null)
+ docker.addArgs("--entrypoint=" + entrypoint);
+
+ docker.addArgs(image);
+ docker.addArgs(arguments.toArray(new String[arguments.size()]));
+
+ ProcessKiller killer = newDockerKiller(newDocker(), containerName, jobLogger);
+ return docker.execute(newInfoLogger(jobLogger), newErrorLogger(jobLogger), null, killer).getReturnCode();
+ }
+
@Override
public boolean execute(LeafExecutable executable, List<Integer> position) {
String stepNames = entryExecutable.getNamesAsString(position);
@@ -233,55 +278,31 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
CommandExecutable commandExecutable = (CommandExecutable) executable;
if (commandExecutable.getImage() == null) {
- throw new ExplicitException("This step should be executed by server shell "
+ throw new ExplicitException("This step can only be executed by server shell "
+ "executor or remote shell executor");
}
Commandline entrypoint = DockerExecutorUtils.getEntrypoint(
hostBuildHome, commandExecutable, hostAuthInfoHome.get() != null);
- String containerName = network + "-step-" + stringifyPosition(position);
- Commandline docker = newDocker();
- docker.clearArgs();
- docker.addArgs("run", "--name=" + containerName, "--network=" + network);
- if (getRunOptions() != null)
- docker.addArgs(StringUtils.parseQuoteTokens(getRunOptions()));
-
- docker.addArgs("-v", getOuterPath(hostBuildHome.getAbsolutePath()) + ":" + containerBuildHome);
- if (workspaceCache.get() != null)
- docker.addArgs("-v", getOuterPath(workspaceCache.get().getAbsolutePath()) + ":" + containerWorkspace);
- for (Map.Entry<CacheInstance, String> entry: cacheAllocations.entrySet()) {
- if (!PathUtils.isCurrent(entry.getValue())) {
- String each = entry.getKey().getDirectory(hostCacheHome).getAbsolutePath();
- String containerCachePath = PathUtils.resolve(containerWorkspace, entry.getValue());
- docker.addArgs("-v", getOuterPath(each) + ":" + containerCachePath);
- }
- }
-
- if (SystemUtils.IS_OS_LINUX)
- docker.addArgs("-v", "/var/run/docker.sock:/var/run/docker.sock");
+ int exitCode = runStepContainer(commandExecutable.getImage(), entrypoint.executable(),
+ entrypoint.arguments(), new HashMap<>(), null, position, commandExecutable.isUseTTY());
- if (hostAuthInfoHome.get() != null) {
- String outerPath = getOuterPath(hostAuthInfoHome.get().getAbsolutePath());
- if (SystemUtils.IS_OS_WINDOWS) {
- docker.addArgs("-v", outerPath + ":C:\\Users\\ContainerAdministrator\\auth-info");
- docker.addArgs("-v", outerPath + ":C:\\Users\\ContainerUser\\auth-info");
- } else {
- docker.addArgs("-v", outerPath + ":/root/auth-info");
- }
+ if (exitCode != 0) {
+ jobLogger.error("Step \"" + stepNames + "\" is failed: Command exited with code " + exitCode);
+ return false;
}
-
- if (commandExecutable.isUseTTY())
- docker.addArgs("-t");
- docker.addArgs("-w", containerWorkspace, "--entrypoint=" + entrypoint.executable());
-
- docker.addArgs(commandExecutable.getImage());
- docker.addArgs(entrypoint.arguments().toArray(new String[0]));
-
- ProcessKiller killer = newDockerKiller(newDocker(), containerName, jobLogger);
- ExecutionResult result = docker.execute(newInfoLogger(jobLogger), newErrorLogger(jobLogger), null, killer);
- if (result.getReturnCode() != 0) {
- jobLogger.error("Step \"" + stepNames + "\" is failed: Command failed with exit code " + result.getReturnCode());
+ } else if (executable instanceof ContainerExecutable) {
+ ContainerExecutable containerExecutable = (ContainerExecutable) executable;
+
+ List<String> arguments = new ArrayList<>();
+ if (containerExecutable.getArgs() != null)
+ arguments.addAll(Arrays.asList(StringUtils.parseQuoteTokens(containerExecutable.getArgs())));
+ int exitCode = runStepContainer(containerExecutable.getImage(), null, arguments,
+ containerExecutable.getEnvMap(), containerExecutable.getWorkingDir(),
+ position, containerExecutable.isUseTTY());
+ if (exitCode != 0) {
+ jobLogger.error("Step \"" + stepNames + "\" is failed: Container exited with code " + exitCode);
return false;
}
} else if (executable instanceof CheckoutExecutable) {
diff --git a/server-plugin/server-plugin-executor-servershell/src/main/java/io/onedev/server/plugin/executor/servershell/ServerShellExecutor.java b/server-plugin/server-plugin-executor-servershell/src/main/java/io/onedev/server/plugin/executor/servershell/ServerShellExecutor.java
index 8074432e12..cc81425809 100644
--- a/server-plugin/server-plugin-executor-servershell/src/main/java/io/onedev/server/plugin/executor/servershell/ServerShellExecutor.java
+++ b/server-plugin/server-plugin-executor-servershell/src/main/java/io/onedev/server/plugin/executor/servershell/ServerShellExecutor.java
@@ -40,6 +40,7 @@ import io.onedev.k8shelper.CheckoutExecutable;
import io.onedev.k8shelper.CloneInfo;
import io.onedev.k8shelper.CommandExecutable;
import io.onedev.k8shelper.CompositeExecutable;
+import io.onedev.k8shelper.ContainerExecutable;
import io.onedev.k8shelper.LeafExecutable;
import io.onedev.k8shelper.LeafHandler;
import io.onedev.k8shelper.ServerExecutable;
@@ -114,21 +115,8 @@ public class ServerShellExecutor extends JobExecutor implements Testable<TestDat
});
- File workspaceCache = null;
- for (Map.Entry<CacheInstance, String> entry: cacheAllocations.entrySet()) {
- if (PathUtils.isCurrent(entry.getValue())) {
- workspaceCache = entry.getKey().getDirectory(cacheHomeDir);
- break;
- }
- }
-
- File workspaceDir;
- if (workspaceCache != null) {
- workspaceDir = workspaceCache;
- } else {
- workspaceDir = new File(buildDir, "workspace");
- FileUtils.createDir(workspaceDir);
- }
+ File workspaceDir = new File(buildDir, "workspace");
+ FileUtils.createDir(workspaceDir);
jobLogger.log("Copying job dependencies...");
jobContext.copyDependencies(workspaceDir);
@@ -150,7 +138,7 @@ public class ServerShellExecutor extends JobExecutor implements Testable<TestDat
if (executable instanceof CommandExecutable) {
CommandExecutable commandExecutable = (CommandExecutable) executable;
if (commandExecutable.getImage() != null) {
- throw new ExplicitException("This step should be executed by server docker executor, "
+ throw new ExplicitException("This step can only be executed by server docker executor, "
+ "remote docker executor, or kubernetes executor");
}
@@ -177,6 +165,8 @@ public class ServerShellExecutor extends JobExecutor implements Testable<TestDat
} catch (IOException e) {
throw new RuntimeException(e);
}
+ } else {
+ throw new ExplicitException("Invalid cache path: " + entry.getValue());
}
}
@@ -188,9 +178,12 @@ public class ServerShellExecutor extends JobExecutor implements Testable<TestDat
ExecutionResult result = interpreter.execute(newInfoLogger(jobLogger), newErrorLogger(jobLogger));
if (result.getReturnCode() != 0) {
- jobLogger.error("Step \"" + stepNames + "\" is failed: Command failed with exit code " + result.getReturnCode());
+ jobLogger.error("Step \"" + stepNames + "\" is failed: Command exited with code " + result.getReturnCode());
return false;
}
+ } else if (executable instanceof ContainerExecutable) {
+ throw new ExplicitException("This step can only be executed by server docker executor, "
+ + "remote docker executor, or kubernetes executor");
} else if (executable instanceof CheckoutExecutable) {
try {
CheckoutExecutable checkoutExecutable = (CheckoutExecutable) executable;
diff --git a/server-plugin/server-plugin-report-checkstyle/src/main/java/io/onedev/server/plugin/report/checkstyle/PublishESLintReportStep.java b/server-plugin/server-plugin-report-checkstyle/src/main/java/io/onedev/server/plugin/report/checkstyle/PublishESLintReportStep.java
index b8c4a1470a..96d5a3bb2d 100644
--- a/server-plugin/server-plugin-report-checkstyle/src/main/java/io/onedev/server/plugin/report/checkstyle/PublishESLintReportStep.java
+++ b/server-plugin/server-plugin-report-checkstyle/src/main/java/io/onedev/server/plugin/report/checkstyle/PublishESLintReportStep.java
@@ -6,7 +6,7 @@ import io.onedev.server.web.editable.annotation.Editable;
import io.onedev.server.web.editable.annotation.Interpolative;
import io.onedev.server.web.editable.annotation.Patterns;
-@Editable(order=300, name="Publish ESLint Report")
+@Editable(order=3000, name="Publish ESLint Report")
public class PublishESLintReportStep extends PublishCheckstyleReportStep {
private static final long serialVersionUID = 1L;
diff --git a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PublishMarkdownReportStep.java b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PublishMarkdownReportStep.java
index 08c1917ce7..39853577be 100644
--- a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PublishMarkdownReportStep.java
+++ b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PublishMarkdownReportStep.java
@@ -18,7 +18,7 @@ import io.onedev.server.model.Build;
import io.onedev.server.web.editable.annotation.Editable;
import io.onedev.server.web.editable.annotation.Interpolative;
-@Editable(order=110, name="Publish Markdown Report")
+@Editable(order=1100, name="Publish Markdown Report")
public class PublishMarkdownReportStep extends PublishReportStep {
private static final long serialVersionUID = 1L;
diff --git a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PublishPullRequestMarkdownReportStep.java b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PublishPullRequestMarkdownReportStep.java
index 4e03164913..485472f53a 100644
--- a/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PublishPullRequestMarkdownReportStep.java
+++ b/server-plugin/server-plugin-report-markdown/src/main/java/io/onedev/server/plugin/report/markdown/PublishPullRequestMarkdownReportStep.java
@@ -18,7 +18,7 @@ import io.onedev.server.model.Build;
import io.onedev.server.web.editable.annotation.Editable;
import io.onedev.server.web.editable.annotation.Interpolative;
-@Editable(order=120, name="Publish Markdown Report (for Pull Request)", description="This report will be displayed in pull request overview "
+@Editable(order=1200, name="Publish Markdown Report (for Pull Request)", description="This report will be displayed in pull request overview "
+ "page if the build belongs to a pull request")
public class PublishPullRequestMarkdownReportStep extends PublishReportStep {
diff --git a/server-product/docker/Dockerfile b/server-product/docker/Dockerfile
index 30c6e42ebb..525e71da70 100644
--- a/server-product/docker/Dockerfile
+++ b/server-product/docker/Dockerfile
@@ -3,7 +3,8 @@ FROM ubuntu:18.04
COPY app /app
COPY entrypoint.sh idle.sh /root/bin/
COPY kubectl /usr/local/bin/
-RUN apt-get update && apt-get install -y locales docker.io iproute2 curl git git-lfs openjdk-8-jre-headless
+COPY docker /usr/local/bin/
+RUN apt-get update && apt-get install -y locales iproute2 curl git git-lfs openjdk-8-jre-headless && rm -rf /var/lib/apt/lists/*
RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
locale-gen
ENV LANG en_US.UTF-8
diff --git a/server-product/docker/docker b/server-product/docker/docker
new file mode 100755
index 0000000000..ee72d7b096
Binary files /dev/null and b/server-product/docker/docker differ