The Unified and Autonomous Development Platform
feat: Improve build image step to be able to generate OCI layout (#1850)
Robin Shen63fdf16bfd88
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java b/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.javaindex 34f68e07d4..cdf440d26b 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java@@ -2013,6 +2013,44 @@ public class BuildSpec implements Serializable, Validatable {stepNode.setTag(new Tag("!OsvVulnerScannerStep"));} else if (stepType.equals("!ScanLicenseViolationsStep")) {stepNode.setTag(new Tag("!OsvLicenseScannerStep"));+ } else if (stepType.equals("!BuildImageStep")) {+ String tags = "";+ for (var itStepTuple = stepNode.getValue().iterator(); itStepTuple.hasNext();) {+ var stepTuple = itStepTuple.next();+ var propName = ((ScalarNode) stepTuple.getKeyNode()).getValue();+ if (propName.equals("tags")) {+ tags = ((ScalarNode)stepTuple.getValueNode()).getValue();+ itStepTuple.remove();+ } else if (propName.equals("publish")) {+ itStepTuple.remove();+ }+ }+ var outputNode = new MappingNode(new Tag("!RegistryOutput"), Lists.newArrayList(), FlowStyle.BLOCK);+ outputNode.getValue().add(new NodeTuple(+ new ScalarNode(Tag.STR, "tags"),+ new ScalarNode(Tag.STR, tags)));+ stepNode.getValue().add(new NodeTuple(+ new ScalarNode(Tag.STR, "output"),+ outputNode+ ));+ } else if (stepType.equals("!BuildImageWithKanikoStep")) {+ String destinations = "";+ for (var itStepTuple = stepNode.getValue().iterator(); itStepTuple.hasNext();) {+ var stepTuple = itStepTuple.next();+ var propName = ((ScalarNode) stepTuple.getKeyNode()).getValue();+ if (propName.equals("destinations")) {+ destinations = ((ScalarNode)stepTuple.getValueNode()).getValue();+ itStepTuple.remove();+ }+ }+ var outputNode = new MappingNode(new Tag("!RegistryOutput"), Lists.newArrayList(), FlowStyle.BLOCK);+ outputNode.getValue().add(new NodeTuple(+ new ScalarNode(Tag.STR, "destinations"),+ new ScalarNode(Tag.STR, destinations)));+ stepNode.getValue().add(new NodeTuple(+ new ScalarNode(Tag.STR, "output"),+ outputNode+ ));}}});diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/BuildImageStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/BuildImageStep.javaindex 39fe67317e..bcb20ad4e4 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/step/BuildImageStep.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/BuildImageStep.java@@ -21,9 +21,10 @@ import java.util.List;import static io.onedev.server.buildspec.step.StepGroup.DOCKER_IMAGE;import static java.util.stream.Collectors.toList;-@Editable(order=160, name="Build Docker Image", group = DOCKER_IMAGE, description="Build and publish docker image with docker buildx. " +- "This step can only be executed by server docker executor or remote docker executor. To build image with " +- "Kubernetes executor, please use kaniko step instead")+@Editable(order=160, name="Build Docker Image", group = DOCKER_IMAGE, description="Build docker image with docker buildx. " ++ "This step can only be executed by server docker executor or remote docker executor, and it uses the buildx " ++ "builder specified in these executors to do the job. To build image with Kubernetes executor, please use kaniko " ++ "step instead")public class BuildImageStep extends Step {private static final long serialVersionUID = 1L;@@ -106,10 +107,9 @@ public class BuildImageStep extends Step {}@Editable(order=1200, group = "More Settings", description="Optionally specify additional options to build image, " +- "separated by spaces. For instance <code>--builder</code> and <code>--platform</code> can be " +- "used to build multi-arch images")+ "separated by spaces. For instance <code>--platform</code> can be used to build multi-arch images")@Interpolative(variableSuggester="suggestVariables")- @ReservedOptions({"--push", "-f", "--file", "(-f|--file)=.*", "-t", "--tag", "(-t|--tag)=.*"})+ @ReservedOptions({"--builder", "--push", "-f", "--file", "(-f|--file)=.*", "-t", "--tag", "(-t|--tag)=.*"})public String getMoreOptions() {return moreOptions;}@@ -147,9 +147,9 @@ public class BuildImageStep extends Step {private String tags;- @Editable(description="Specify full tag of the image, for instance <tt>myorg/myrepo:latest</tt>, "- + "<tt>myorg/myrepo:1.0.0</tt>, or <tt>onedev.example.com/myproject/myrepo:1.0.0</tt>. "- + "Multiple tags should be separated with space.<br>")+ @Editable(description="Specify full tag of the image, for instance <tt>myorg/myrepo:latest</tt>, " ++ "<tt>myorg/myrepo:1.0.0</tt>, or <tt>onedev.example.com/myproject/myrepo:1.0.0</tt>. " ++ "Multiple tags should be separated with space")@Interpolative(variableSuggester="suggestVariables")@NotEmptypublic String getTags() {@@ -170,7 +170,7 @@ public class BuildImageStep extends Step {}}- @Editable(order=200, name="Export to OCI layout")+ @Editable(order=200, name="Export as OCI layout")public static class OCIOutput implements Output {private static final long serialVersionUID = 1L;@@ -189,6 +189,10 @@ public class BuildImageStep extends Step {this.destPath = destPath;}+ static List<InputSuggestion> suggestVariables(String matchWith) {+ return BuildSpec.suggestVariables(matchWith, true, true, false);+ }+@Overridepublic BuildImageFacade.Output getFacade() {return new BuildImageFacade.OCIOutput(destPath);diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/BuildImageWithKanikoStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/BuildImageWithKanikoStep.javaindex 430fb1c1ce..c7d81a6ae8 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/step/BuildImageWithKanikoStep.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/BuildImageWithKanikoStep.java@@ -7,7 +7,6 @@ import io.onedev.k8shelper.CommandFacade;import io.onedev.server.OneDev;import io.onedev.server.annotation.*;import io.onedev.server.buildspec.BuildSpec;-import io.onedev.server.buildspec.step.CommandStep;import io.onedev.server.buildspec.step.commandinterpreter.DefaultInterpreter;import io.onedev.server.buildspec.step.commandinterpreter.Interpreter;import io.onedev.server.entitymanager.SettingManager;@@ -17,13 +16,16 @@ import io.onedev.server.model.support.administration.jobexecutor.RegistryLoginAwimport io.onedev.server.util.UrlUtils;import javax.validation.constraints.NotEmpty;+import javax.validation.constraints.NotNull;+import java.io.Serializable;+import java.util.ArrayList;import java.util.List;import static io.onedev.agent.DockerExecutorUtils.buildDockerConfig;import static io.onedev.server.buildspec.step.StepGroup.DOCKER_IMAGE;import static java.util.stream.Collectors.toList;-@Editable(order=200, name="Build Docker Image (Kaniko)", group = DOCKER_IMAGE, description="Build and publish docker image with Kaniko. " ++@Editable(order=200, name="Build Docker Image (Kaniko)", group = DOCKER_IMAGE, description="Build docker image with kaniko. " +"This step needs to be executed by server docker executor, remote docker executor, or Kubernetes executor")public class BuildImageWithKanikoStep extends CommandStep {@@ -31,7 +33,7 @@ public class BuildImageWithKanikoStep extends CommandStep {private String buildContext;- private String destinations;+ private Output output = new RegistryOutput();private String trustCertificates;@@ -72,19 +74,16 @@ public class BuildImageWithKanikoStep extends CommandStep {this.buildContext = buildContext;}- @Editable(order=300, description="Specify destinations, for instance <tt>myorg/myrepo:latest</tt>, "- + "<tt>myorg/myrepo:1.0.0</tt>, or <tt>myregistry:5000/myorg/myrepo:1.0.0</tt>. "- + "Multiple destinations should be separated with space.<br>")- @Interpolative(variableSuggester="suggestVariables")- @NotEmpty- public String getDestinations() {- return destinations;+ @Editable(order=300)+ @NotNull+ public Output getOutput() {+ return output;}- public void setDestinations(String destinations) {- this.destinations = destinations;+ public void setOutput(Output output) {+ this.output = output;}-+@Editable(order=1000, name="Certificates to Trust", group = "More Settings", placeholder = "Base64 encoded PEM format, starting with " +"-----BEGIN CERTIFICATE----- and ending with -----END CERTIFICATE-----",description = "Specify certificates to trust if you are using self-signed certificates for your docker registries")@@ -117,10 +116,9 @@ public class BuildImageWithKanikoStep extends CommandStep {"<code>" + server + "</code>";}- @Editable(order=1200, group="More Settings", description="Optionally specify additional options to build image, " +- "separated by spaces")+ @Editable(order=1200, group="More Settings", description="Optionally specify <a href='https://github.com/GoogleContainerTools/kaniko?tab=readme-ov-file#additional-flags' target='_blank'>additional options</a> of kaniko")@Interpolative(variableSuggester="suggestVariables")- @ReservedOptions({"(--context)=.*", "(--destination)=.*"})+ @ReservedOptions({"(--context)=.*", "(--destination)=.*", "(--oci-layout-path)=.*", "--no-push"})public String getMoreOptions() {return moreOptions;}@@ -129,10 +127,6 @@ public class BuildImageWithKanikoStep extends CommandStep {this.moreOptions = moreOptions;}- static List<InputSuggestion> suggestVariables(String matchWith) {- return BuildSpec.suggestVariables(matchWith, true, true, false);- }-@Overridepublic Interpreter getInterpreter() {return new DefaultInterpreter() {@@ -161,8 +155,9 @@ public class BuildImageWithKanikoStep extends CommandStep {commandsBuilder.append(" --context=\"/onedev-build/workspace/").append(getBuildContext()).append("\"");elsecommandsBuilder.append(" --context=/onedev-build/workspace");- for (var destination: StringUtils.splitAndTrim(getDestinations(), " "))- commandsBuilder.append(" --destination=").append(destination);++ commandsBuilder.append(" ").append(getOutput().getOptions());+if (getMoreOptions() != null)commandsBuilder.append(" ").append(getMoreOptions());@@ -173,4 +168,74 @@ public class BuildImageWithKanikoStep extends CommandStep {};}+ @Editable+ public static interface Output extends Serializable {++ String getOptions();++ }++ @Editable(order=100, name="Push to Container registry")+ public static class RegistryOutput implements Output {++ private static final long serialVersionUID = 1L;++ private String destinations;++ @Editable(order=300, description="Specify destinations, for instance <tt>myorg/myrepo:latest</tt>, "+ + "<tt>myorg/myrepo:1.0.0</tt>, or <tt>myregistry:5000/myorg/myrepo:1.0.0</tt>. "+ + "Multiple destinations should be separated with space.<br>")+ @Interpolative(variableSuggester="suggestVariables")+ @NotEmpty+ public String getDestinations() {+ return destinations;+ }++ public void setDestinations(String destinations) {+ this.destinations = destinations;+ }++ static List<InputSuggestion> suggestVariables(String matchWith) {+ return BuildSpec.suggestVariables(matchWith, true, true, false);+ }++ @Override+ public String getOptions() {+ var options = new ArrayList<String>();+ for (var destination: StringUtils.splitAndTrim(getDestinations(), " "))+ options.add("--destination=" + destination);+ return StringUtils.join(options, " ");+ }++ }++ @Editable(order=200, name="Export as OCI layout")+ public static class OCIOutput implements Output {++ private static final long serialVersionUID = 1L;++ private String destPath;++ @Editable(name="OCI Layout Directory", description = "Specify path relative to <a href='https://docs.onedev.io/concepts#job-workspace' target='_blank'>job workspace</a> to store OCI layout")+ @Interpolative(variableSuggester="suggestVariables")+ @NotEmpty+ public String getDestPath() {+ return destPath;+ }++ public void setDestPath(String destPath) {+ this.destPath = destPath;+ }++ static List<InputSuggestion> suggestVariables(String matchWith) {+ return BuildSpec.suggestVariables(matchWith, true, true, false);+ }++ @Override+ public String getOptions() {+ return "--no-push --oci-layout-path /onedev-build/workspace/" + getDestPath();+ }++ }+}diff --git a/server-core/src/main/java/io/onedev/server/data/migration/DataMigrator.java b/server-core/src/main/java/io/onedev/server/data/migration/DataMigrator.javaindex 4b38384c70..f7616c2864 100644--- a/server-core/src/main/java/io/onedev/server/data/migration/DataMigrator.java+++ b/server-core/src/main/java/io/onedev/server/data/migration/DataMigrator.java@@ -6296,4 +6296,26 @@ public class DataMigrator {}}+ private void migrate160(File dataDir, Stack<Integer> versions) {+ for (File file : dataDir.listFiles()) {+ if (file.getName().startsWith("Settings.xml")) {+ VersionedXmlDoc dom = VersionedXmlDoc.fromFile(file);+ for (Element element : dom.getRootElement().elements()) {+ String key = element.elementTextTrim("key");+ if (key.equals("JOB_EXECUTORS")) {+ Element valueElement = element.element("value");+ if (valueElement != null) {+ for (Element executorElement : valueElement.elements()) {+ if (executorElement.getName().contains("ServerDockerExecutor")+ || executorElement.getName().contains("RemoteDockerExecutor")) {+ executorElement.addElement("dockerBuilder").setText("onedev");+ }+ }+ }+ }+ }+ dom.writeToFile(file, false);+ }+ }+ }}\ No newline at end of filediff --git a/server-core/src/main/java/io/onedev/server/git/CommandUtils.java b/server-core/src/main/java/io/onedev/server/git/CommandUtils.javaindex 53f599ccaf..4cd3fe220e 100644--- a/server-core/src/main/java/io/onedev/server/git/CommandUtils.java+++ b/server-core/src/main/java/io/onedev/server/git/CommandUtils.java@@ -151,7 +151,7 @@ public class CommandUtils {UploadPackCommand upload = new UploadPackCommand(gitDir, stdin, stdout, stderr, environments);upload.statelessRpc(true).protocol(protocol);result = upload.run();- result.setStderr(stderr.getMessage());+ result.setErrorMessage(stderr.getMessage());if (result.getReturnCode() != 0 && !toleratedErrors.get())throw result.buildException();@@ -172,7 +172,7 @@ public class CommandUtils {ReceivePackCommand receive = new ReceivePackCommand(gitDir, stdin, stdout, stderr, environments);receive.statelessRpc(true).protocol(protocol);ExecutionResult result = receive.run();- result.setStderr(stderr.getMessage());+ result.setErrorMessage(stderr.getMessage());result.checkReturnCode();}diff --git a/server-core/src/main/java/io/onedev/server/terminal/CommandlineShell.java b/server-core/src/main/java/io/onedev/server/terminal/CommandlineShell.javaindex 541b122601..ff7c75fefc 100644--- a/server-core/src/main/java/io/onedev/server/terminal/CommandlineShell.java+++ b/server-core/src/main/java/io/onedev/server/terminal/CommandlineShell.java@@ -1,24 +1,24 @@package io.onedev.server.terminal;+import io.onedev.commons.utils.ExceptionUtils;+import io.onedev.commons.utils.ExplicitException;+import io.onedev.commons.utils.ImmediateFuture;+import io.onedev.commons.utils.command.*;+import io.onedev.commons.utils.command.PtyMode.ResizeSupport;+import io.onedev.server.OneDev;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;+import java.io.IOException;+import java.io.InputStream;import java.io.OutputStream;import java.nio.charset.StandardCharsets;import java.util.concurrent.ExecutorService;import java.util.concurrent.Future;+import java.util.concurrent.atomic.AtomicReference;+import java.util.function.Function;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--import io.onedev.commons.utils.ExceptionUtils;-import io.onedev.commons.utils.ExplicitException;-import io.onedev.commons.utils.command.Commandline;-import io.onedev.commons.utils.command.ExecutionResult;-import io.onedev.commons.utils.command.ExposeOutputStream;-import io.onedev.commons.utils.command.ProcessTreeKiller;-import io.onedev.commons.utils.command.PtyMode;-import io.onedev.commons.utils.command.PtyMode.ResizeSupport;-import io.onedev.commons.utils.command.PumpInputToOutput;-import io.onedev.server.OneDev;+import static org.apache.commons.io.IOUtils.closeQuietly;public class CommandlineShell implements Shell {@@ -26,7 +26,7 @@ public class CommandlineShell implements Shell {private final PtyMode ptyMode;- private final ExposeOutputStream shellInput;+ private volatile OutputStream shellStdin;private final Future<?> execution;@@ -34,49 +34,61 @@ public class CommandlineShell implements Shell {ptyMode = new PtyMode();cmdline.ptyMode(ptyMode);- shellInput = new ExposeOutputStream();execution = OneDev.getInstance(ExecutorService.class).submit(new Runnable() {@Overridepublic void run() {try {- PumpInputToOutput outputHandler = new PumpInputToOutput(new OutputStream() {-- @Override- public void write(byte[] b, int off, int len) throws IOException {- terminal.sendOutput(new String(b, off, len, StandardCharsets.UTF_8));- }-- @Override- public void write(int b) throws IOException {- throw new UnsupportedOperationException();- }-- });- PumpInputToOutput errorHandler = new PumpInputToOutput(new OutputStream() {-- @Override- public void write(byte[] b, int off, int len) throws IOException {- terminal.sendError(new String(b, off, len, StandardCharsets.UTF_8));- }-- @Override- public void write(int b) throws IOException {- throw new UnsupportedOperationException();- }-- });+ var stdoutHolder = new AtomicReference<InputStream>(null);+ Function<InputStream, Future<?>> stdoutHandler = is -> {+ stdoutHolder.set(is);+ return StreamPumper.pump(is, new OutputStream() {++ @Override+ public void write(byte[] b, int off, int len) {+ terminal.sendOutput(new String(b, off, len, StandardCharsets.UTF_8));+ }++ @Override+ public void write(int b) {+ throw new UnsupportedOperationException();+ }++ });+ };++ var stderrHolder = new AtomicReference<InputStream>(null);+ Function<InputStream, Future<?>> stderrHandler = is -> {+ stderrHolder.set(is);+ return StreamPumper.pump(is, new OutputStream() {++ @Override+ public void write(byte[] b, int off, int len) {+ terminal.sendError(new String(b, off, len, StandardCharsets.UTF_8));+ }++ @Override+ public void write(int b) {+ throw new UnsupportedOperationException();+ }++ });+ };+cmdline.processKiller(new ProcessTreeKiller() {@Overridepublic void kill(Process process, String executionId) {- outputHandler.close();- errorHandler.close();+ closeQuietly(stdoutHolder.get());+ closeQuietly(stderrHolder.get());super.kill(process, executionId);}});- ExecutionResult result = cmdline.execute(outputHandler, errorHandler, shellInput);+ ExecutionResult result = cmdline.execute(stdoutHandler, stderrHandler, os -> {+ shellStdin = os;+ return new ImmediateFuture<Void>(null);+ });if (result.getReturnCode() != 0)terminal.sendError("Shell exited");else@@ -93,7 +105,10 @@ public class CommandlineShell implements Shell {logger.error("Error running shell", e);terminal.sendError("Error running shell, check server log for details");}- }+ } finally {+ closeQuietly(shellStdin);+ shellStdin = null;+ }}});@@ -102,9 +117,13 @@ public class CommandlineShell implements Shell {@Overridepublic void sendInput(String input) {- try {- shellInput.write(input);- } catch (IOException e) {+ var shellStdinCopy = shellStdin;+ if (shellStdinCopy != null) {+ try {+ shellStdinCopy.write(input.getBytes(StandardCharsets.UTF_8));+ } catch (IOException e) {+ throw new RuntimeException(e);+ }}}diff --git a/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java b/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.javaindex 826c54be92..b82b0d9050 100644--- a/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java+++ b/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java@@ -122,7 +122,9 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw}@Editable(order=515, group="More Settings", name="Buildx Builder", description = "Specify dockerx builder used to " +- "build docker image")+ "build docker image. OneDev will create the builder automatically if it does not exist. Check " ++ "<a href='https://docs.onedev.io/tutorials/cicd/insecure-docker-registry' target='_blank'>this tutorial</a> " ++ "on how to customize the builder for instance to allow publishing to insecure registries")@NotEmptypublic String getDockerBuilder() {return dockerBuilder;