The Unified and Autonomous Development Platform
feat: Implement migration for Docker builders in settings
Robin Shen886d7dc58cf6
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java b/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.javaindex 3e25873374..d132d147b2 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java@@ -14,12 +14,6 @@ import java.util.Stack;import java.util.function.Consumer;import java.util.function.Function;-import jakarta.validation.ConstraintValidatorContext;-import jakarta.validation.ConstraintViolation;-import jakarta.validation.Valid;-import jakarta.validation.ValidationException;-import jakarta.validation.Validator;-import org.apache.commons.lang3.SerializationUtils;import org.jspecify.annotations.Nullable;import org.yaml.snakeyaml.DumperOptions.FlowStyle;@@ -38,7 +32,6 @@ import com.google.common.collect.Lists;import io.onedev.commons.codeassist.InputSuggestion;import io.onedev.commons.utils.ExceptionUtils;-import io.onedev.commons.utils.ExplicitException;import io.onedev.commons.utils.StringUtils;import io.onedev.commons.utils.WordUtils;import io.onedev.server.OneDev;@@ -62,6 +55,11 @@ import io.onedev.server.validation.Validatable;import io.onedev.server.web.page.project.blob.ProjectBlobPage;import io.onedev.server.web.util.SuggestionUtils;import io.onedev.server.web.util.WicketUtils;+import jakarta.validation.ConstraintValidatorContext;+import jakarta.validation.ConstraintViolation;+import jakarta.validation.Valid;+import jakarta.validation.ValidationException;+import jakarta.validation.Validator;@Editable@ClassValidatingdiff --git a/server-core/src/main/java/io/onedev/server/data/migration/DataMigrator.java b/server-core/src/main/java/io/onedev/server/data/migration/DataMigrator.javaindex 89e0635bf4..2edd770aec 100644--- a/server-core/src/main/java/io/onedev/server/data/migration/DataMigrator.java+++ b/server-core/src/main/java/io/onedev/server/data/migration/DataMigrator.java@@ -9529,4 +9529,28 @@ public class DataMigrator {// Recreate the schema to restore longblob columns on MySQL/MariaDB (OD-3157).}+ private void migrate248(File dataDir, Stack<Integer> versions) {+ for (File file : dataDir.listFiles()) {+ if (file.getName().startsWith("Settings.xml")) {+ var dom = VersionedXmlDoc.fromFile(file);+ for (Element element : dom.getRootElement().elements()) {+ if ("JOB_EXECUTORS".equals(element.elementTextTrim("key"))) {+ var valueElement = element.element("value");+ if (valueElement != null) {+ for (Element executorElement : valueElement.elements()) {+ if (executorElement.getName().endsWith(".ServerDockerExecutor")+ || executorElement.getName().endsWith(".RemoteDockerExecutor")) {+ var builderElement = executorElement.element("dockerBuilder");+ if (builderElement != null)+ builderElement.detach();+ }+ }+ }+ }+ }+ dom.writeToFile(file, false);+ }+ }+ }+}diff --git a/server-core/src/test/java/io/onedev/server/data/migration/DockerBuilderMigrationTest.java b/server-core/src/test/java/io/onedev/server/data/migration/DockerBuilderMigrationTest.javanew file mode 100644index 0000000000..dfdff2b8fc--- /dev/null+++ b/server-core/src/test/java/io/onedev/server/data/migration/DockerBuilderMigrationTest.java@@ -0,0 +1,64 @@+package io.onedev.server.data.migration;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertTrue;++import java.nio.file.Path;+import java.util.List;++import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.io.TempDir;++class DockerBuilderMigrationTest {++ @TempDir+ Path directory;++ @Test+ void clearsDefaultAndCustomBuildersAcrossSettingsBatches() {+ var executorTypes = List.of(+ "io.onedev.server.plugin.executor.serverdocker.ServerDockerExecutor",+ "io.onedev.server.plugin.executor.remotedocker.RemoteDockerExecutor");+ for (int batch = 0; batch < executorTypes.size(); batch++) {+ var doc = new VersionedXmlDoc();+ var list = doc.addElement("list");+ var setting = list.addElement("io.onedev.server.model.Setting");+ setting.addElement("key").setText("JOB_EXECUTORS");+ var executors = setting.addElement("value");+ for (var builder : List.of("onedev", "custom-builder", "")) {+ var executor = executors.addElement(executorTypes.get(batch));+ executor.addElement("name").setText("docker-" + builder);+ if (!builder.isEmpty())+ executor.addElement("dockerBuilder").setText(builder);+ executor.addElement("buildOptions").setText("--no-cache");+ }+ var otherSetting = list.addElement("io.onedev.server.model.Setting");+ otherSetting.addElement("key").setText("OTHER");+ otherSetting.addElement("value").addElement("dockerBuilder").setText("preserved");+ var emptySetting = list.addElement("io.onedev.server.model.Setting");+ emptySetting.addElement("key").setText("JOB_EXECUTORS");+ doc.writeToFile(directory.resolve("Settings.xml" + (batch == 0 ? "" : "." + batch)).toFile(), false);+ }++ assertTrue(MigrationHelper.migrate("247", new DataMigrator(), directory.toFile()));++ for (int batch = 0; batch < executorTypes.size(); batch++) {+ var doc = VersionedXmlDoc.fromFile(+ directory.resolve("Settings.xml" + (batch == 0 ? "" : "." + batch)).toFile());+ var settings = doc.getRootElement().elements();+ var executors = settings.get(0).element("value").elements();+ assertEquals(3, executors.size());+ for (int i = 0; i < executors.size(); i++) {+ var executor = executors.get(i);+ assertEquals(executorTypes.get(batch), executor.getName());+ assertEquals(List.of("docker-onedev", "docker-custom-builder", "docker-").get(i),+ executor.elementText("name"));+ assertNull(executor.element("dockerBuilder"));+ assertEquals("--no-cache", executor.elementText("buildOptions"));+ }+ assertEquals("preserved", settings.get(1).element("value").elementText("dockerBuilder"));+ assertNull(settings.get(2).element("value"));+ }+ }+}diff --git a/server-plugin/server-plugin-executor-remotedocker/src/main/java/io/onedev/server/plugin/executor/remotedocker/RemoteDockerExecutor.java b/server-plugin/server-plugin-executor-remotedocker/src/main/java/io/onedev/server/plugin/executor/remotedocker/RemoteDockerExecutor.javaindex bdd57e4ddf..18d41075d0 100644--- a/server-plugin/server-plugin-executor-remotedocker/src/main/java/io/onedev/server/plugin/executor/remotedocker/RemoteDockerExecutor.java+++ b/server-plugin/server-plugin-executor-remotedocker/src/main/java/io/onedev/server/plugin/executor/remotedocker/RemoteDockerExecutor.java@@ -109,7 +109,7 @@ public class RemoteDockerExecutor extends ServerDockerExecutor {var dockerSettings = new JobDockerSettings(isMountDockerSock(), getDockerSockPath(),getCpuLimit(), getMemoryLimit(), getRunOptions(), registryLogins,- isAlwaysPullImage(), getDockerBuilder(), getNetworkOptions(), getBuildOptions());+ isAlwaysPullImage(), getEffectiveDockerBuilder(), getNetworkOptions(), getBuildOptions());DockerJobData jobData = new DockerJobData(jobToken, getName(), jobContext.getProjectPath(),jobContext.getProjectId(), jobContext.getRefName(), jobContext.getCommitId().name(),jobContext.getBuildNumber(), jobContext.getSubmitSequence(), jobContext.getActions(),@@ -171,4 +171,4 @@ public class RemoteDockerExecutor extends ServerDockerExecutor {jobLogger);}-}\ No newline at end of file+}diff --git a/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java b/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.javaindex e40590a30a..6f802b6c64 100644--- a/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java+++ b/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java@@ -107,7 +107,7 @@ public class ServerDockerExecutor extends JobExecutor implements DockerAware, Teprivate String dockerSockPath;- private String dockerBuilder = "onedev";+ private String dockerBuilder;private String networkOptions;@@ -174,11 +174,13 @@ public class ServerDockerExecutor extends JobExecutor implements DockerAware, Tethis.dockerSockPath = dockerSockPath;}- @Editable(order=515, group="More Settings", name="Buildx Builder", description = "Specify dockerx builder used to " +- "build docker image. OneDev will create the builder automatically if it does not exist. Check " ++ @Editable(order=515, group="More Settings", name="Buildx Builder", placeholder="executor-<executor name>",+ description = "Optionally specify the Buildx builder used to build docker images. " ++ "Leave empty to use <code>executor-<executor name></code>. OneDev will create the builder " ++ "automatically if it does not exist. Jobs using the same builder can read or modify each other's " ++ "build cache, so only share a builder among trusted jobs. Jobs using this executor still share its builder. Check " +"<a href='https://docs.onedev.io/tutorials/cicd/insecure-docker-registry' target='_blank'>this tutorial</a> " +"on how to customize the builder for instance to allow publishing to insecure registries")- @NotEmptypublic String getDockerBuilder() {return dockerBuilder;}@@ -186,6 +188,10 @@ public class ServerDockerExecutor extends JobExecutor implements DockerAware, Tepublic void setDockerBuilder(String dockerBuilder) {this.dockerBuilder = dockerBuilder;}++ protected String getEffectiveDockerBuilder() {+ return getDockerBuilder() != null ? getDockerBuilder() : "executor-" + getName();+ }@Editable(order=600, group="Security Settings", description = "Whether or not to always pull image when " +"run container or build images. This option should be enabled to avoid images being replaced by " +@@ -495,7 +501,7 @@ public class ServerDockerExecutor extends JobExecutor implements DockerAware, Tevar docker = newDocker();var registryLogins = merge(buildImageFacade.getRegistryLogins(), getRegistryLogins(jobToken));callWithRegistryLogins(docker, registryLogins, () -> {- JobUtils.buildImage(docker, getDockerBuilder(), getBuildOptions(), buildImageFacade, buildDir,+ JobUtils.buildImage(docker, getEffectiveDockerBuilder(), getBuildOptions(), buildImageFacade, buildDir,isAlwaysPullImage(), jobLogger);return null;});@@ -511,7 +517,7 @@ public class ServerDockerExecutor extends JobExecutor implements DockerAware, Tevar pruneBuilderCacheFacade = (PruneBuilderCacheFacade) facade;var docker = newDocker();callWithRegistryLogins(docker, new ArrayList<>(), () -> {- JobUtils.pruneBuilderCache(docker, getDockerBuilder(), pruneBuilderCacheFacade,+ JobUtils.pruneBuilderCache(docker, getEffectiveDockerBuilder(), pruneBuilderCacheFacade,buildDir, jobLogger);return null;});diff --git a/server-product/system/incompatibilities/incompatibilities.md b/server-product/system/incompatibilities/incompatibilities.mdindex f77fabbca5..3e305187c6 100644--- a/server-product/system/incompatibilities/incompatibilities.md+++ b/server-product/system/incompatibilities/incompatibilities.md@@ -6,6 +6,10 @@ The `More Options` setting (`moreOptions` in build specs) of the build image steExisting step options are removed when build specs are migrated and are not copied to executors automatically. Before upgrading, record any options you need to retain and ask an administrator to configure them on the appropriate Docker job executor after upgrading. Executor options apply to all build image steps using that executor.+The `Buildx Builder` setting of all server and remote Docker job executors is cleared during upgrade, including custom builder names. When unset, it now defaults to `executor-<executor name>` instead of `onedev`. Image builds and cache pruning use this executor-specific builder, so existing builder caches and custom configuration are no longer used by default. Before upgrading, record any custom builder settings and reconfigure the new builders or explicitly restore a builder name afterward if needed.++Jobs using the same Buildx builder can read or modify each other's build cache. Only share a builder among trusted jobs; jobs using the same executor still share its builder. Use separate executors and builders for mutually untrusted jobs.+# 16.8.0### CI/CD