Skip to content

The Unified and Autonomous Development Platform

Able to wait for finish of pod execution and get the logs

robinb7a2ce17f3ed
diff --git a/server-core/src/main/java/io/onedev/server/util/Maps.java b/server-core/src/main/java/io/onedev/server/util/Maps.java
index b745b8d673..4e88d1895b 100644
--- a/server-core/src/main/java/io/onedev/server/util/Maps.java
+++ b/server-core/src/main/java/io/onedev/server/util/Maps.java
@@ -8,19 +8,19 @@ import com.google.common.base.Preconditions;
public class Maps {
- public static Map<Object, Object> newLinkedHashMap(Object...args) {
- Map<Object, Object> map = new LinkedHashMap<>();
+ public static <T> Map<T, T> newLinkedHashMap(@SuppressWarnings("unchecked") T...args) {
+ Map<T, T> map = new LinkedHashMap<>();
initMap(map, args);
return map;
}
- public static Map<Object, Object> newHashMap(Object...args) {
- Map<Object, Object> map = new HashMap<>();
+ public static <T> Map<T, T> newHashMap(@SuppressWarnings("unchecked") T...args) {
+ Map<T, T> map = new HashMap<>();
initMap(map, args);
return map;
}
- private static void initMap(Map<Object, Object> map, Object...args) {
+ private static <T> void initMap(Map<T, T> map, @SuppressWarnings("unchecked") T...args) {
Preconditions.checkArgument(args.length % 2 == 0, "Arguments should be key/value pairs");
for (int i=0; i<args.length/2; i++)
map.put(args[i*2], args[i*2+1]);
diff --git a/server-plugin/server-plugin-kubernetes/src/main/java/io/onedev/server/plugin/kubernetes/KubernetesExecutor.java b/server-plugin/server-plugin-kubernetes/src/main/java/io/onedev/server/plugin/kubernetes/KubernetesExecutor.java
index 1c4e7a87a8..10846e8649 100644
--- a/server-plugin/server-plugin-kubernetes/src/main/java/io/onedev/server/plugin/kubernetes/KubernetesExecutor.java
+++ b/server-plugin/server-plugin-kubernetes/src/main/java/io/onedev/server/plugin/kubernetes/KubernetesExecutor.java
@@ -10,6 +10,7 @@ import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.StringTokenizer;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
@@ -26,7 +27,9 @@ import com.google.common.collect.Lists;
import io.onedev.commons.utils.FileUtils;
import io.onedev.commons.utils.StringUtils;
import io.onedev.commons.utils.command.Commandline;
+import io.onedev.commons.utils.command.ExecuteResult;
import io.onedev.commons.utils.command.LineConsumer;
+import io.onedev.server.OneException;
import io.onedev.server.ci.job.cache.JobCache;
import io.onedev.server.model.support.JobExecutor;
import io.onedev.server.model.support.SourceSnapshot;
@@ -271,14 +274,54 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
return data;
}
- private String getPodNodeOS(String podImage, Logger logger) {
+ private String getOS(Logger logger) {
+ logger.info("Checking OS...");
+ Commandline kubectl = newKubeCtl();
+ kubectl.addArgs("get", "nodes", "-o=jsonpath={..nodeInfo.operatingSystem}");
+ for (NodeSelectorEntry entry: getNodeSelector())
+ kubectl.addArgs("-l", entry.getLabelName() + "=" + entry.getLabelValue());
+
+ AtomicReference<String> osRef = new AtomicReference<>(null);
+ kubectl.execute(new LineConsumer() {
+
+ @Override
+ public void consume(String line) {
+ osRef.set(line);
+ }
+
+ }, new LineConsumer() {
+
+ @Override
+ public void consume(String line) {
+ logger.error(line);
+ }
+
+ }).checkReturnCode();
+
+ return Preconditions.checkNotNull(osRef.get(), "No applicable working nodes for this executor");
+ }
+
+ @Override
+ public void test(TestData testData) {
createNamespaceIfNotExist(logger);
+ String os = getOS(logger);
+
Map<String, Object> podSpec = new LinkedHashMap<>();
- podSpec.put("containers", Lists.<Object>newArrayList(
- Maps.newLinkedHashMap(
- "name", "test",
- "image", podImage)));
+ Map<Object, Object> containerSpec = Maps.newHashMap(
+ "name", "test",
+ "image", testData.getDockerImage());
+
+ if (os.equalsIgnoreCase("linux")) {
+ containerSpec.put("command", Lists.newArrayList("sh"));
+ containerSpec.put("args", Lists.newArrayList("-c", "echo hello from container"));
+ } else {
+ containerSpec.put("command", Lists.newArrayList("cmd"));
+ containerSpec.put("args", Lists.newArrayList("/c", "echo hello from container"));
+ }
+
+ podSpec.put("containers", Lists.<Object>newArrayList(containerSpec));
+
Map<String, String> nodeSelectorData = getNodeSelectorData();
if (!nodeSelectorData.isEmpty())
podSpec.put("nodeSelector", nodeSelectorData);
@@ -297,16 +340,45 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
"spec", podSpec);
String podName = createResource(podData, logger);
+ try {
+ waitForPod(podName, logger);
+ } finally {
+ deleteResource("pod", podName, logger);
+ }
+ }
+
+ private void waitForPod(String podName, Logger logger) {
+ Thread thread = Thread.currentThread();
+
+ AtomicBoolean podStartedRef = new AtomicBoolean(false);
+ AtomicReference<String> podErrorRef = new AtomicReference<String>(null);
+ Commandline kubectl = newKubeCtl();
+ kubectl.addArgs("get", "event", "-n", getNamespace(), "--no-headers",
+ "--field-selector", "involvedObject.name=" + podName, "--watch");
try {
- AtomicReference<String> nodeNameRef = new AtomicReference<>(null);
- Commandline kubectl = newKubeCtl();
- kubectl.addArgs("get", "pods/" + podName, "-n", getNamespace(), "-o=jsonpath={..spec.nodeName}");
kubectl.execute(new LineConsumer() {
@Override
public void consume(String line) {
- nodeNameRef.set(line);
+ StringTokenizer tokenizer = new StringTokenizer(line);
+ tokenizer.nextToken();
+ String type = tokenizer.nextToken();
+ tokenizer.nextToken();
+ tokenizer.nextToken();
+ String message = tokenizer.nextToken("\n").trim();
+ if (type.equals("Normal"))
+ logger.info(message);
+ else
+ logger.error(message);
+
+ if (!type.equals("Normal") && !message.contains("Insufficient cpu")) {
+ podErrorRef.set(message);
+ thread.interrupt();
+ } else if (message.startsWith("Started container")) {
+ podStartedRef.set(true);
+ thread.interrupt();
+ }
}
}, new LineConsumer() {
@@ -316,42 +388,55 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData
logger.error(line);
}
- }).checkReturnCode();
-
- Preconditions.checkNotNull(nodeNameRef.get());
-
- AtomicReference<String> nodeOSRef = new AtomicReference<String>(null);
+ });
- kubectl = newKubeCtl();
- kubectl.addArgs("get", "nodes/" + nodeNameRef.get(), "-o=jsonpath={..nodeInfo.operatingSystem}");
- kubectl.execute(new LineConsumer() {
-
- @Override
- public void consume(String line) {
- nodeOSRef.set(line);
+ throw new OneException("Unexpected end of pod event watching");
+ } catch (Exception e) {
+ if (e.getCause() instanceof InterruptedException) {
+ if (podStartedRef.get()) {
+ kubectl = newKubeCtl();
+ kubectl.addArgs("logs", podName, "-n", getNamespace(), "--follow");
+ while (true) {
+ AtomicReference<Boolean> containerCreatingRef = new AtomicReference<Boolean>(false);
+ ExecuteResult result = kubectl.execute(new LineConsumer() {
+
+ @Override
+ public void consume(String line) {
+ logger.info(line);
+ }
+
+ }, new LineConsumer() {
+
+ @Override
+ public void consume(String line) {
+ if (line.contains("is waiting to start: ContainerCreating"))
+ containerCreatingRef.set(true);
+ else
+ logger.error(line);
+ }
+
+ });
+ if (containerCreatingRef.get()) {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e2) {
+ }
+ } else {
+ result.checkReturnCode();
+ break;
+ }
+ }
+ } else if (podErrorRef.get() != null) {
+ throw new OneException(podErrorRef.get());
+ } else {
+ throw e;
}
-
- }, new LineConsumer() {
-
- @Override
- public void consume(String line) {
- logger.error(line);
- }
-
- }).checkReturnCode();
-
- return nodeOSRef.get();
- } finally {
- deleteResource("pod", podName, logger);
+ } else {
+ throw e;
+ }
}
}
- @Override
- public void test(TestData data) {
- String podNodeOS = getPodNodeOS(data.getDockerImage(), logger);
- logger.info("OS of the node running test pod is " + podNodeOS);
- }
-
@Editable
public static class NodeSelectorEntry implements Serializable {