The Unified and Autonomous Development Platform
Able to wait for finish of pod execution and get the logs
robinb7a2ce17f3ed
diff --git a/server-core/src/main/java/io/onedev/server/util/Maps.java b/server-core/src/main/java/io/onedev/server/util/Maps.javaindex b745b8d673..4e88d1895b 100644--- a/server-core/src/main/java/io/onedev/server/util/Maps.java+++ b/server-core/src/main/java/io/onedev/server/util/Maps.java@@ -8,19 +8,19 @@ import com.google.common.base.Preconditions;public class Maps {- public static Map<Object, Object> newLinkedHashMap(Object...args) {- Map<Object, Object> map = new LinkedHashMap<>();+ public static <T> Map<T, T> newLinkedHashMap(@SuppressWarnings("unchecked") T...args) {+ Map<T, T> map = new LinkedHashMap<>();initMap(map, args);return map;}- public static Map<Object, Object> newHashMap(Object...args) {- Map<Object, Object> map = new HashMap<>();+ public static <T> Map<T, T> newHashMap(@SuppressWarnings("unchecked") T...args) {+ Map<T, T> map = new HashMap<>();initMap(map, args);return map;}- private static void initMap(Map<Object, Object> map, Object...args) {+ private static <T> void initMap(Map<T, T> map, @SuppressWarnings("unchecked") T...args) {Preconditions.checkArgument(args.length % 2 == 0, "Arguments should be key/value pairs");for (int i=0; i<args.length/2; i++)map.put(args[i*2], args[i*2+1]);diff --git a/server-plugin/server-plugin-kubernetes/src/main/java/io/onedev/server/plugin/kubernetes/KubernetesExecutor.java b/server-plugin/server-plugin-kubernetes/src/main/java/io/onedev/server/plugin/kubernetes/KubernetesExecutor.javaindex 1c4e7a87a8..10846e8649 100644--- a/server-plugin/server-plugin-kubernetes/src/main/java/io/onedev/server/plugin/kubernetes/KubernetesExecutor.java+++ b/server-plugin/server-plugin-kubernetes/src/main/java/io/onedev/server/plugin/kubernetes/KubernetesExecutor.java@@ -10,6 +10,7 @@ import java.util.Collection;import java.util.LinkedHashMap;import java.util.List;import java.util.Map;+import java.util.StringTokenizer;import java.util.concurrent.atomic.AtomicBoolean;import java.util.concurrent.atomic.AtomicReference;@@ -26,7 +27,9 @@ import com.google.common.collect.Lists;import io.onedev.commons.utils.FileUtils;import io.onedev.commons.utils.StringUtils;import io.onedev.commons.utils.command.Commandline;+import io.onedev.commons.utils.command.ExecuteResult;import io.onedev.commons.utils.command.LineConsumer;+import io.onedev.server.OneException;import io.onedev.server.ci.job.cache.JobCache;import io.onedev.server.model.support.JobExecutor;import io.onedev.server.model.support.SourceSnapshot;@@ -271,14 +274,54 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestDatareturn data;}- private String getPodNodeOS(String podImage, Logger logger) {+ private String getOS(Logger logger) {+ logger.info("Checking OS...");+ Commandline kubectl = newKubeCtl();+ kubectl.addArgs("get", "nodes", "-o=jsonpath={..nodeInfo.operatingSystem}");+ for (NodeSelectorEntry entry: getNodeSelector())+ kubectl.addArgs("-l", entry.getLabelName() + "=" + entry.getLabelValue());++ AtomicReference<String> osRef = new AtomicReference<>(null);+ kubectl.execute(new LineConsumer() {++ @Override+ public void consume(String line) {+ osRef.set(line);+ }++ }, new LineConsumer() {++ @Override+ public void consume(String line) {+ logger.error(line);+ }++ }).checkReturnCode();++ return Preconditions.checkNotNull(osRef.get(), "No applicable working nodes for this executor");+ }++ @Override+ public void test(TestData testData) {createNamespaceIfNotExist(logger);+ String os = getOS(logger);+Map<String, Object> podSpec = new LinkedHashMap<>();- podSpec.put("containers", Lists.<Object>newArrayList(- Maps.newLinkedHashMap(- "name", "test",- "image", podImage)));+ Map<Object, Object> containerSpec = Maps.newHashMap(+ "name", "test",+ "image", testData.getDockerImage());++ if (os.equalsIgnoreCase("linux")) {+ containerSpec.put("command", Lists.newArrayList("sh"));+ containerSpec.put("args", Lists.newArrayList("-c", "echo hello from container"));+ } else {+ containerSpec.put("command", Lists.newArrayList("cmd"));+ containerSpec.put("args", Lists.newArrayList("/c", "echo hello from container"));+ }++ podSpec.put("containers", Lists.<Object>newArrayList(containerSpec));+Map<String, String> nodeSelectorData = getNodeSelectorData();if (!nodeSelectorData.isEmpty())podSpec.put("nodeSelector", nodeSelectorData);@@ -297,16 +340,45 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData"spec", podSpec);String podName = createResource(podData, logger);+ try {+ waitForPod(podName, logger);+ } finally {+ deleteResource("pod", podName, logger);+ }+ }++ private void waitForPod(String podName, Logger logger) {+ Thread thread = Thread.currentThread();++ AtomicBoolean podStartedRef = new AtomicBoolean(false);+ AtomicReference<String> podErrorRef = new AtomicReference<String>(null);+ Commandline kubectl = newKubeCtl();+ kubectl.addArgs("get", "event", "-n", getNamespace(), "--no-headers",+ "--field-selector", "involvedObject.name=" + podName, "--watch");try {- AtomicReference<String> nodeNameRef = new AtomicReference<>(null);- Commandline kubectl = newKubeCtl();- kubectl.addArgs("get", "pods/" + podName, "-n", getNamespace(), "-o=jsonpath={..spec.nodeName}");kubectl.execute(new LineConsumer() {@Overridepublic void consume(String line) {- nodeNameRef.set(line);+ StringTokenizer tokenizer = new StringTokenizer(line);+ tokenizer.nextToken();+ String type = tokenizer.nextToken();+ tokenizer.nextToken();+ tokenizer.nextToken();+ String message = tokenizer.nextToken("\n").trim();+ if (type.equals("Normal"))+ logger.info(message);+ else+ logger.error(message);++ if (!type.equals("Normal") && !message.contains("Insufficient cpu")) {+ podErrorRef.set(message);+ thread.interrupt();+ } else if (message.startsWith("Started container")) {+ podStartedRef.set(true);+ thread.interrupt();+ }}}, new LineConsumer() {@@ -316,42 +388,55 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestDatalogger.error(line);}- }).checkReturnCode();-- Preconditions.checkNotNull(nodeNameRef.get());-- AtomicReference<String> nodeOSRef = new AtomicReference<String>(null);+ });- kubectl = newKubeCtl();- kubectl.addArgs("get", "nodes/" + nodeNameRef.get(), "-o=jsonpath={..nodeInfo.operatingSystem}");- kubectl.execute(new LineConsumer() {-- @Override- public void consume(String line) {- nodeOSRef.set(line);+ throw new OneException("Unexpected end of pod event watching");+ } catch (Exception e) {+ if (e.getCause() instanceof InterruptedException) {+ if (podStartedRef.get()) {+ kubectl = newKubeCtl();+ kubectl.addArgs("logs", podName, "-n", getNamespace(), "--follow");+ while (true) {+ AtomicReference<Boolean> containerCreatingRef = new AtomicReference<Boolean>(false);+ ExecuteResult result = kubectl.execute(new LineConsumer() {++ @Override+ public void consume(String line) {+ logger.info(line);+ }++ }, new LineConsumer() {++ @Override+ public void consume(String line) {+ if (line.contains("is waiting to start: ContainerCreating"))+ containerCreatingRef.set(true);+ else+ logger.error(line);+ }++ });+ if (containerCreatingRef.get()) {+ try {+ Thread.sleep(1000);+ } catch (InterruptedException e2) {+ }+ } else {+ result.checkReturnCode();+ break;+ }+ }+ } else if (podErrorRef.get() != null) {+ throw new OneException(podErrorRef.get());+ } else {+ throw e;}-- }, new LineConsumer() {-- @Override- public void consume(String line) {- logger.error(line);- }-- }).checkReturnCode();-- return nodeOSRef.get();- } finally {- deleteResource("pod", podName, logger);+ } else {+ throw e;+ }}}- @Override- public void test(TestData data) {- String podNodeOS = getPodNodeOS(data.getDockerImage(), logger);- logger.info("OS of the node running test pod is " + podNodeOS);- }-@Editablepublic static class NodeSelectorEntry implements Serializable {