Skip to content

The Unified and Autonomous Development Platform

feat: Improve build image step to be able to generate OCI layout (#1850)

Robin Shen63fdf16bfd88
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java b/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java
index 34f68e07d4..cdf440d26b 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java
@@ -2013,6 +2013,44 @@ public class BuildSpec implements Serializable, Validatable {
stepNode.setTag(new Tag("!OsvVulnerScannerStep"));
} else if (stepType.equals("!ScanLicenseViolationsStep")) {
stepNode.setTag(new Tag("!OsvLicenseScannerStep"));
+ } else if (stepType.equals("!BuildImageStep")) {
+ String tags = "";
+ for (var itStepTuple = stepNode.getValue().iterator(); itStepTuple.hasNext();) {
+ var stepTuple = itStepTuple.next();
+ var propName = ((ScalarNode) stepTuple.getKeyNode()).getValue();
+ if (propName.equals("tags")) {
+ tags = ((ScalarNode)stepTuple.getValueNode()).getValue();
+ itStepTuple.remove();
+ } else if (propName.equals("publish")) {
+ itStepTuple.remove();
+ }
+ }
+ var outputNode = new MappingNode(new Tag("!RegistryOutput"), Lists.newArrayList(), FlowStyle.BLOCK);
+ outputNode.getValue().add(new NodeTuple(
+ new ScalarNode(Tag.STR, "tags"),
+ new ScalarNode(Tag.STR, tags)));
+ stepNode.getValue().add(new NodeTuple(
+ new ScalarNode(Tag.STR, "output"),
+ outputNode
+ ));
+ } else if (stepType.equals("!BuildImageWithKanikoStep")) {
+ String destinations = "";
+ for (var itStepTuple = stepNode.getValue().iterator(); itStepTuple.hasNext();) {
+ var stepTuple = itStepTuple.next();
+ var propName = ((ScalarNode) stepTuple.getKeyNode()).getValue();
+ if (propName.equals("destinations")) {
+ destinations = ((ScalarNode)stepTuple.getValueNode()).getValue();
+ itStepTuple.remove();
+ }
+ }
+ var outputNode = new MappingNode(new Tag("!RegistryOutput"), Lists.newArrayList(), FlowStyle.BLOCK);
+ outputNode.getValue().add(new NodeTuple(
+ new ScalarNode(Tag.STR, "destinations"),
+ new ScalarNode(Tag.STR, destinations)));
+ stepNode.getValue().add(new NodeTuple(
+ new ScalarNode(Tag.STR, "output"),
+ outputNode
+ ));
}
}
});
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/BuildImageStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/BuildImageStep.java
index 39fe67317e..bcb20ad4e4 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/step/BuildImageStep.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/BuildImageStep.java
@@ -21,9 +21,10 @@ import java.util.List;
import static io.onedev.server.buildspec.step.StepGroup.DOCKER_IMAGE;
import static java.util.stream.Collectors.toList;
-@Editable(order=160, name="Build Docker Image", group = DOCKER_IMAGE, description="Build and publish docker image with docker buildx. " +
- "This step can only be executed by server docker executor or remote docker executor. To build image with " +
- "Kubernetes executor, please use kaniko step instead")
+@Editable(order=160, name="Build Docker Image", group = DOCKER_IMAGE, description="Build docker image with docker buildx. " +
+ "This step can only be executed by server docker executor or remote docker executor, and it uses the buildx " +
+ "builder specified in these executors to do the job. To build image with Kubernetes executor, please use kaniko " +
+ "step instead")
public class BuildImageStep extends Step {
private static final long serialVersionUID = 1L;
@@ -106,10 +107,9 @@ public class BuildImageStep extends Step {
}
@Editable(order=1200, group = "More Settings", description="Optionally specify additional options to build image, " +
- "separated by spaces. For instance <code>--builder</code> and <code>--platform</code> can be " +
- "used to build multi-arch images")
+ "separated by spaces. For instance <code>--platform</code> can be used to build multi-arch images")
@Interpolative(variableSuggester="suggestVariables")
- @ReservedOptions({"--push", "-f", "--file", "(-f|--file)=.*", "-t", "--tag", "(-t|--tag)=.*"})
+ @ReservedOptions({"--builder", "--push", "-f", "--file", "(-f|--file)=.*", "-t", "--tag", "(-t|--tag)=.*"})
public String getMoreOptions() {
return moreOptions;
}
@@ -147,9 +147,9 @@ public class BuildImageStep extends Step {
private String tags;
- @Editable(description="Specify full tag of the image, for instance <tt>myorg/myrepo:latest</tt>, "
- + "<tt>myorg/myrepo:1.0.0</tt>, or <tt>onedev.example.com/myproject/myrepo:1.0.0</tt>. "
- + "Multiple tags should be separated with space.<br>")
+ @Editable(description="Specify full tag of the image, for instance <tt>myorg/myrepo:latest</tt>, " +
+ "<tt>myorg/myrepo:1.0.0</tt>, or <tt>onedev.example.com/myproject/myrepo:1.0.0</tt>. " +
+ "Multiple tags should be separated with space")
@Interpolative(variableSuggester="suggestVariables")
@NotEmpty
public String getTags() {
@@ -170,7 +170,7 @@ public class BuildImageStep extends Step {
}
}
- @Editable(order=200, name="Export to OCI layout")
+ @Editable(order=200, name="Export as OCI layout")
public static class OCIOutput implements Output {
private static final long serialVersionUID = 1L;
@@ -189,6 +189,10 @@ public class BuildImageStep extends Step {
this.destPath = destPath;
}
+ static List<InputSuggestion> suggestVariables(String matchWith) {
+ return BuildSpec.suggestVariables(matchWith, true, true, false);
+ }
+
@Override
public BuildImageFacade.Output getFacade() {
return new BuildImageFacade.OCIOutput(destPath);
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/BuildImageWithKanikoStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/BuildImageWithKanikoStep.java
index 430fb1c1ce..c7d81a6ae8 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/step/BuildImageWithKanikoStep.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/BuildImageWithKanikoStep.java
@@ -7,7 +7,6 @@ import io.onedev.k8shelper.CommandFacade;
import io.onedev.server.OneDev;
import io.onedev.server.annotation.*;
import io.onedev.server.buildspec.BuildSpec;
-import io.onedev.server.buildspec.step.CommandStep;
import io.onedev.server.buildspec.step.commandinterpreter.DefaultInterpreter;
import io.onedev.server.buildspec.step.commandinterpreter.Interpreter;
import io.onedev.server.entitymanager.SettingManager;
@@ -17,13 +16,16 @@ import io.onedev.server.model.support.administration.jobexecutor.RegistryLoginAw
import io.onedev.server.util.UrlUtils;
import javax.validation.constraints.NotEmpty;
+import javax.validation.constraints.NotNull;
+import java.io.Serializable;
+import java.util.ArrayList;
import java.util.List;
import static io.onedev.agent.DockerExecutorUtils.buildDockerConfig;
import static io.onedev.server.buildspec.step.StepGroup.DOCKER_IMAGE;
import static java.util.stream.Collectors.toList;
-@Editable(order=200, name="Build Docker Image (Kaniko)", group = DOCKER_IMAGE, description="Build and publish docker image with Kaniko. " +
+@Editable(order=200, name="Build Docker Image (Kaniko)", group = DOCKER_IMAGE, description="Build docker image with kaniko. " +
"This step needs to be executed by server docker executor, remote docker executor, or Kubernetes executor")
public class BuildImageWithKanikoStep extends CommandStep {
@@ -31,7 +33,7 @@ public class BuildImageWithKanikoStep extends CommandStep {
private String buildContext;
- private String destinations;
+ private Output output = new RegistryOutput();
private String trustCertificates;
@@ -72,19 +74,16 @@ public class BuildImageWithKanikoStep extends CommandStep {
this.buildContext = buildContext;
}
- @Editable(order=300, description="Specify destinations, for instance <tt>myorg/myrepo:latest</tt>, "
- + "<tt>myorg/myrepo:1.0.0</tt>, or <tt>myregistry:5000/myorg/myrepo:1.0.0</tt>. "
- + "Multiple destinations should be separated with space.<br>")
- @Interpolative(variableSuggester="suggestVariables")
- @NotEmpty
- public String getDestinations() {
- return destinations;
+ @Editable(order=300)
+ @NotNull
+ public Output getOutput() {
+ return output;
}
- public void setDestinations(String destinations) {
- this.destinations = destinations;
+ public void setOutput(Output output) {
+ this.output = output;
}
-
+
@Editable(order=1000, name="Certificates to Trust", group = "More Settings", placeholder = "Base64 encoded PEM format, starting with " +
"-----BEGIN CERTIFICATE----- and ending with -----END CERTIFICATE-----",
description = "Specify certificates to trust if you are using self-signed certificates for your docker registries")
@@ -117,10 +116,9 @@ public class BuildImageWithKanikoStep extends CommandStep {
"<code>" + server + "</code>";
}
- @Editable(order=1200, group="More Settings", description="Optionally specify additional options to build image, " +
- "separated by spaces")
+ @Editable(order=1200, group="More Settings", description="Optionally specify <a href='https://github.com/GoogleContainerTools/kaniko?tab=readme-ov-file#additional-flags' target='_blank'>additional options</a> of kaniko")
@Interpolative(variableSuggester="suggestVariables")
- @ReservedOptions({"(--context)=.*", "(--destination)=.*"})
+ @ReservedOptions({"(--context)=.*", "(--destination)=.*", "(--oci-layout-path)=.*", "--no-push"})
public String getMoreOptions() {
return moreOptions;
}
@@ -129,10 +127,6 @@ public class BuildImageWithKanikoStep extends CommandStep {
this.moreOptions = moreOptions;
}
- static List<InputSuggestion> suggestVariables(String matchWith) {
- return BuildSpec.suggestVariables(matchWith, true, true, false);
- }
-
@Override
public Interpreter getInterpreter() {
return new DefaultInterpreter() {
@@ -161,8 +155,9 @@ public class BuildImageWithKanikoStep extends CommandStep {
commandsBuilder.append(" --context=\"/onedev-build/workspace/").append(getBuildContext()).append("\"");
else
commandsBuilder.append(" --context=/onedev-build/workspace");
- for (var destination: StringUtils.splitAndTrim(getDestinations(), " "))
- commandsBuilder.append(" --destination=").append(destination);
+
+ commandsBuilder.append(" ").append(getOutput().getOptions());
+
if (getMoreOptions() != null)
commandsBuilder.append(" ").append(getMoreOptions());
@@ -173,4 +168,74 @@ public class BuildImageWithKanikoStep extends CommandStep {
};
}
+ @Editable
+ public static interface Output extends Serializable {
+
+ String getOptions();
+
+ }
+
+ @Editable(order=100, name="Push to Container registry")
+ public static class RegistryOutput implements Output {
+
+ private static final long serialVersionUID = 1L;
+
+ private String destinations;
+
+ @Editable(order=300, description="Specify destinations, for instance <tt>myorg/myrepo:latest</tt>, "
+ + "<tt>myorg/myrepo:1.0.0</tt>, or <tt>myregistry:5000/myorg/myrepo:1.0.0</tt>. "
+ + "Multiple destinations should be separated with space.<br>")
+ @Interpolative(variableSuggester="suggestVariables")
+ @NotEmpty
+ public String getDestinations() {
+ return destinations;
+ }
+
+ public void setDestinations(String destinations) {
+ this.destinations = destinations;
+ }
+
+ static List<InputSuggestion> suggestVariables(String matchWith) {
+ return BuildSpec.suggestVariables(matchWith, true, true, false);
+ }
+
+ @Override
+ public String getOptions() {
+ var options = new ArrayList<String>();
+ for (var destination: StringUtils.splitAndTrim(getDestinations(), " "))
+ options.add("--destination=" + destination);
+ return StringUtils.join(options, " ");
+ }
+
+ }
+
+ @Editable(order=200, name="Export as OCI layout")
+ public static class OCIOutput implements Output {
+
+ private static final long serialVersionUID = 1L;
+
+ private String destPath;
+
+ @Editable(name="OCI Layout Directory", description = "Specify path relative to <a href='https://docs.onedev.io/concepts#job-workspace' target='_blank'>job workspace</a> to store OCI layout")
+ @Interpolative(variableSuggester="suggestVariables")
+ @NotEmpty
+ public String getDestPath() {
+ return destPath;
+ }
+
+ public void setDestPath(String destPath) {
+ this.destPath = destPath;
+ }
+
+ static List<InputSuggestion> suggestVariables(String matchWith) {
+ return BuildSpec.suggestVariables(matchWith, true, true, false);
+ }
+
+ @Override
+ public String getOptions() {
+ return "--no-push --oci-layout-path /onedev-build/workspace/" + getDestPath();
+ }
+
+ }
+
}
diff --git a/server-core/src/main/java/io/onedev/server/data/migration/DataMigrator.java b/server-core/src/main/java/io/onedev/server/data/migration/DataMigrator.java
index 4b38384c70..f7616c2864 100644
--- a/server-core/src/main/java/io/onedev/server/data/migration/DataMigrator.java
+++ b/server-core/src/main/java/io/onedev/server/data/migration/DataMigrator.java
@@ -6296,4 +6296,26 @@ public class DataMigrator {
}
}
+ private void migrate160(File dataDir, Stack<Integer> versions) {
+ for (File file : dataDir.listFiles()) {
+ if (file.getName().startsWith("Settings.xml")) {
+ VersionedXmlDoc dom = VersionedXmlDoc.fromFile(file);
+ for (Element element : dom.getRootElement().elements()) {
+ String key = element.elementTextTrim("key");
+ if (key.equals("JOB_EXECUTORS")) {
+ Element valueElement = element.element("value");
+ if (valueElement != null) {
+ for (Element executorElement : valueElement.elements()) {
+ if (executorElement.getName().contains("ServerDockerExecutor")
+ || executorElement.getName().contains("RemoteDockerExecutor")) {
+ executorElement.addElement("dockerBuilder").setText("onedev");
+ }
+ }
+ }
+ }
+ }
+ dom.writeToFile(file, false);
+ }
+ }
+ }
}
\ No newline at end of file
diff --git a/server-core/src/main/java/io/onedev/server/git/CommandUtils.java b/server-core/src/main/java/io/onedev/server/git/CommandUtils.java
index 53f599ccaf..4cd3fe220e 100644
--- a/server-core/src/main/java/io/onedev/server/git/CommandUtils.java
+++ b/server-core/src/main/java/io/onedev/server/git/CommandUtils.java
@@ -151,7 +151,7 @@ public class CommandUtils {
UploadPackCommand upload = new UploadPackCommand(gitDir, stdin, stdout, stderr, environments);
upload.statelessRpc(true).protocol(protocol);
result = upload.run();
- result.setStderr(stderr.getMessage());
+ result.setErrorMessage(stderr.getMessage());
if (result.getReturnCode() != 0 && !toleratedErrors.get())
throw result.buildException();
@@ -172,7 +172,7 @@ public class CommandUtils {
ReceivePackCommand receive = new ReceivePackCommand(gitDir, stdin, stdout, stderr, environments);
receive.statelessRpc(true).protocol(protocol);
ExecutionResult result = receive.run();
- result.setStderr(stderr.getMessage());
+ result.setErrorMessage(stderr.getMessage());
result.checkReturnCode();
}
diff --git a/server-core/src/main/java/io/onedev/server/terminal/CommandlineShell.java b/server-core/src/main/java/io/onedev/server/terminal/CommandlineShell.java
index 541b122601..ff7c75fefc 100644
--- a/server-core/src/main/java/io/onedev/server/terminal/CommandlineShell.java
+++ b/server-core/src/main/java/io/onedev/server/terminal/CommandlineShell.java
@@ -1,24 +1,24 @@
package io.onedev.server.terminal;
+import io.onedev.commons.utils.ExceptionUtils;
+import io.onedev.commons.utils.ExplicitException;
+import io.onedev.commons.utils.ImmediateFuture;
+import io.onedev.commons.utils.command.*;
+import io.onedev.commons.utils.command.PtyMode.ResizeSupport;
+import io.onedev.server.OneDev;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.IOException;
+import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Function;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import io.onedev.commons.utils.ExceptionUtils;
-import io.onedev.commons.utils.ExplicitException;
-import io.onedev.commons.utils.command.Commandline;
-import io.onedev.commons.utils.command.ExecutionResult;
-import io.onedev.commons.utils.command.ExposeOutputStream;
-import io.onedev.commons.utils.command.ProcessTreeKiller;
-import io.onedev.commons.utils.command.PtyMode;
-import io.onedev.commons.utils.command.PtyMode.ResizeSupport;
-import io.onedev.commons.utils.command.PumpInputToOutput;
-import io.onedev.server.OneDev;
+import static org.apache.commons.io.IOUtils.closeQuietly;
public class CommandlineShell implements Shell {
@@ -26,7 +26,7 @@ public class CommandlineShell implements Shell {
private final PtyMode ptyMode;
- private final ExposeOutputStream shellInput;
+ private volatile OutputStream shellStdin;
private final Future<?> execution;
@@ -34,49 +34,61 @@ public class CommandlineShell implements Shell {
ptyMode = new PtyMode();
cmdline.ptyMode(ptyMode);
- shellInput = new ExposeOutputStream();
execution = OneDev.getInstance(ExecutorService.class).submit(new Runnable() {
@Override
public void run() {
try {
- PumpInputToOutput outputHandler = new PumpInputToOutput(new OutputStream() {
-
- @Override
- public void write(byte[] b, int off, int len) throws IOException {
- terminal.sendOutput(new String(b, off, len, StandardCharsets.UTF_8));
- }
-
- @Override
- public void write(int b) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- });
- PumpInputToOutput errorHandler = new PumpInputToOutput(new OutputStream() {
-
- @Override
- public void write(byte[] b, int off, int len) throws IOException {
- terminal.sendError(new String(b, off, len, StandardCharsets.UTF_8));
- }
-
- @Override
- public void write(int b) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- });
+ var stdoutHolder = new AtomicReference<InputStream>(null);
+ Function<InputStream, Future<?>> stdoutHandler = is -> {
+ stdoutHolder.set(is);
+ return StreamPumper.pump(is, new OutputStream() {
+
+ @Override
+ public void write(byte[] b, int off, int len) {
+ terminal.sendOutput(new String(b, off, len, StandardCharsets.UTF_8));
+ }
+
+ @Override
+ public void write(int b) {
+ throw new UnsupportedOperationException();
+ }
+
+ });
+ };
+
+ var stderrHolder = new AtomicReference<InputStream>(null);
+ Function<InputStream, Future<?>> stderrHandler = is -> {
+ stderrHolder.set(is);
+ return StreamPumper.pump(is, new OutputStream() {
+
+ @Override
+ public void write(byte[] b, int off, int len) {
+ terminal.sendError(new String(b, off, len, StandardCharsets.UTF_8));
+ }
+
+ @Override
+ public void write(int b) {
+ throw new UnsupportedOperationException();
+ }
+
+ });
+ };
+
cmdline.processKiller(new ProcessTreeKiller() {
@Override
public void kill(Process process, String executionId) {
- outputHandler.close();
- errorHandler.close();
+ closeQuietly(stdoutHolder.get());
+ closeQuietly(stderrHolder.get());
super.kill(process, executionId);
}
});
- ExecutionResult result = cmdline.execute(outputHandler, errorHandler, shellInput);
+ ExecutionResult result = cmdline.execute(stdoutHandler, stderrHandler, os -> {
+ shellStdin = os;
+ return new ImmediateFuture<Void>(null);
+ });
if (result.getReturnCode() != 0)
terminal.sendError("Shell exited");
else
@@ -93,7 +105,10 @@ public class CommandlineShell implements Shell {
logger.error("Error running shell", e);
terminal.sendError("Error running shell, check server log for details");
}
- }
+ } finally {
+ closeQuietly(shellStdin);
+ shellStdin = null;
+ }
}
});
@@ -102,9 +117,13 @@ public class CommandlineShell implements Shell {
@Override
public void sendInput(String input) {
- try {
- shellInput.write(input);
- } catch (IOException e) {
+ var shellStdinCopy = shellStdin;
+ if (shellStdinCopy != null) {
+ try {
+ shellStdinCopy.write(input.getBytes(StandardCharsets.UTF_8));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
}
}
diff --git a/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java b/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java
index 826c54be92..b82b0d9050 100644
--- a/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java
+++ b/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java
@@ -122,7 +122,9 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw
}
@Editable(order=515, group="More Settings", name="Buildx Builder", description = "Specify dockerx builder used to " +
- "build docker image")
+ "build docker image. OneDev will create the builder automatically if it does not exist. Check " +
+ "<a href='https://docs.onedev.io/tutorials/cicd/insecure-docker-registry' target='_blank'>this tutorial</a> " +
+ "on how to customize the builder for instance to allow publishing to insecure registries")
@NotEmpty
public String getDockerBuilder() {
return dockerBuilder;