Skip to content

The Unified and Autonomous Development Platform

feat: Implement migration for Docker builders in settings

Robin Shen886d7dc58cf6
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java b/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java
index 3e25873374..d132d147b2 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java
@@ -14,12 +14,6 @@ import java.util.Stack;
import java.util.function.Consumer;
import java.util.function.Function;
-import jakarta.validation.ConstraintValidatorContext;
-import jakarta.validation.ConstraintViolation;
-import jakarta.validation.Valid;
-import jakarta.validation.ValidationException;
-import jakarta.validation.Validator;
-
import org.apache.commons.lang3.SerializationUtils;
import org.jspecify.annotations.Nullable;
import org.yaml.snakeyaml.DumperOptions.FlowStyle;
@@ -38,7 +32,6 @@ import com.google.common.collect.Lists;
import io.onedev.commons.codeassist.InputSuggestion;
import io.onedev.commons.utils.ExceptionUtils;
-import io.onedev.commons.utils.ExplicitException;
import io.onedev.commons.utils.StringUtils;
import io.onedev.commons.utils.WordUtils;
import io.onedev.server.OneDev;
@@ -62,6 +55,11 @@ import io.onedev.server.validation.Validatable;
import io.onedev.server.web.page.project.blob.ProjectBlobPage;
import io.onedev.server.web.util.SuggestionUtils;
import io.onedev.server.web.util.WicketUtils;
+import jakarta.validation.ConstraintValidatorContext;
+import jakarta.validation.ConstraintViolation;
+import jakarta.validation.Valid;
+import jakarta.validation.ValidationException;
+import jakarta.validation.Validator;
@Editable
@ClassValidating
diff --git a/server-core/src/main/java/io/onedev/server/data/migration/DataMigrator.java b/server-core/src/main/java/io/onedev/server/data/migration/DataMigrator.java
index 89e0635bf4..2edd770aec 100644
--- a/server-core/src/main/java/io/onedev/server/data/migration/DataMigrator.java
+++ b/server-core/src/main/java/io/onedev/server/data/migration/DataMigrator.java
@@ -9529,4 +9529,28 @@ public class DataMigrator {
// Recreate the schema to restore longblob columns on MySQL/MariaDB (OD-3157).
}
+ private void migrate248(File dataDir, Stack<Integer> versions) {
+ for (File file : dataDir.listFiles()) {
+ if (file.getName().startsWith("Settings.xml")) {
+ var dom = VersionedXmlDoc.fromFile(file);
+ for (Element element : dom.getRootElement().elements()) {
+ if ("JOB_EXECUTORS".equals(element.elementTextTrim("key"))) {
+ var valueElement = element.element("value");
+ if (valueElement != null) {
+ for (Element executorElement : valueElement.elements()) {
+ if (executorElement.getName().endsWith(".ServerDockerExecutor")
+ || executorElement.getName().endsWith(".RemoteDockerExecutor")) {
+ var builderElement = executorElement.element("dockerBuilder");
+ if (builderElement != null)
+ builderElement.detach();
+ }
+ }
+ }
+ }
+ }
+ dom.writeToFile(file, false);
+ }
+ }
+ }
+
}
diff --git a/server-core/src/test/java/io/onedev/server/data/migration/DockerBuilderMigrationTest.java b/server-core/src/test/java/io/onedev/server/data/migration/DockerBuilderMigrationTest.java
new file mode 100644
index 0000000000..dfdff2b8fc
--- /dev/null
+++ b/server-core/src/test/java/io/onedev/server/data/migration/DockerBuilderMigrationTest.java
@@ -0,0 +1,64 @@
+package io.onedev.server.data.migration;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.nio.file.Path;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+class DockerBuilderMigrationTest {
+
+ @TempDir
+ Path directory;
+
+ @Test
+ void clearsDefaultAndCustomBuildersAcrossSettingsBatches() {
+ var executorTypes = List.of(
+ "io.onedev.server.plugin.executor.serverdocker.ServerDockerExecutor",
+ "io.onedev.server.plugin.executor.remotedocker.RemoteDockerExecutor");
+ for (int batch = 0; batch < executorTypes.size(); batch++) {
+ var doc = new VersionedXmlDoc();
+ var list = doc.addElement("list");
+ var setting = list.addElement("io.onedev.server.model.Setting");
+ setting.addElement("key").setText("JOB_EXECUTORS");
+ var executors = setting.addElement("value");
+ for (var builder : List.of("onedev", "custom-builder", "")) {
+ var executor = executors.addElement(executorTypes.get(batch));
+ executor.addElement("name").setText("docker-" + builder);
+ if (!builder.isEmpty())
+ executor.addElement("dockerBuilder").setText(builder);
+ executor.addElement("buildOptions").setText("--no-cache");
+ }
+ var otherSetting = list.addElement("io.onedev.server.model.Setting");
+ otherSetting.addElement("key").setText("OTHER");
+ otherSetting.addElement("value").addElement("dockerBuilder").setText("preserved");
+ var emptySetting = list.addElement("io.onedev.server.model.Setting");
+ emptySetting.addElement("key").setText("JOB_EXECUTORS");
+ doc.writeToFile(directory.resolve("Settings.xml" + (batch == 0 ? "" : "." + batch)).toFile(), false);
+ }
+
+ assertTrue(MigrationHelper.migrate("247", new DataMigrator(), directory.toFile()));
+
+ for (int batch = 0; batch < executorTypes.size(); batch++) {
+ var doc = VersionedXmlDoc.fromFile(
+ directory.resolve("Settings.xml" + (batch == 0 ? "" : "." + batch)).toFile());
+ var settings = doc.getRootElement().elements();
+ var executors = settings.get(0).element("value").elements();
+ assertEquals(3, executors.size());
+ for (int i = 0; i < executors.size(); i++) {
+ var executor = executors.get(i);
+ assertEquals(executorTypes.get(batch), executor.getName());
+ assertEquals(List.of("docker-onedev", "docker-custom-builder", "docker-").get(i),
+ executor.elementText("name"));
+ assertNull(executor.element("dockerBuilder"));
+ assertEquals("--no-cache", executor.elementText("buildOptions"));
+ }
+ assertEquals("preserved", settings.get(1).element("value").elementText("dockerBuilder"));
+ assertNull(settings.get(2).element("value"));
+ }
+ }
+}
diff --git a/server-plugin/server-plugin-executor-remotedocker/src/main/java/io/onedev/server/plugin/executor/remotedocker/RemoteDockerExecutor.java b/server-plugin/server-plugin-executor-remotedocker/src/main/java/io/onedev/server/plugin/executor/remotedocker/RemoteDockerExecutor.java
index bdd57e4ddf..18d41075d0 100644
--- a/server-plugin/server-plugin-executor-remotedocker/src/main/java/io/onedev/server/plugin/executor/remotedocker/RemoteDockerExecutor.java
+++ b/server-plugin/server-plugin-executor-remotedocker/src/main/java/io/onedev/server/plugin/executor/remotedocker/RemoteDockerExecutor.java
@@ -109,7 +109,7 @@ public class RemoteDockerExecutor extends ServerDockerExecutor {
var dockerSettings = new JobDockerSettings(isMountDockerSock(), getDockerSockPath(),
getCpuLimit(), getMemoryLimit(), getRunOptions(), registryLogins,
- isAlwaysPullImage(), getDockerBuilder(), getNetworkOptions(), getBuildOptions());
+ isAlwaysPullImage(), getEffectiveDockerBuilder(), getNetworkOptions(), getBuildOptions());
DockerJobData jobData = new DockerJobData(jobToken, getName(), jobContext.getProjectPath(),
jobContext.getProjectId(), jobContext.getRefName(), jobContext.getCommitId().name(),
jobContext.getBuildNumber(), jobContext.getSubmitSequence(), jobContext.getActions(),
@@ -171,4 +171,4 @@ public class RemoteDockerExecutor extends ServerDockerExecutor {
jobLogger);
}
-}
\ No newline at end of file
+}
diff --git a/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java b/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java
index e40590a30a..6f802b6c64 100644
--- a/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java
+++ b/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java
@@ -107,7 +107,7 @@ public class ServerDockerExecutor extends JobExecutor implements DockerAware, Te
private String dockerSockPath;
- private String dockerBuilder = "onedev";
+ private String dockerBuilder;
private String networkOptions;
@@ -174,11 +174,13 @@ public class ServerDockerExecutor extends JobExecutor implements DockerAware, Te
this.dockerSockPath = dockerSockPath;
}
- @Editable(order=515, group="More Settings", name="Buildx Builder", description = "Specify dockerx builder used to " +
- "build docker image. OneDev will create the builder automatically if it does not exist. Check " +
+ @Editable(order=515, group="More Settings", name="Buildx Builder", placeholder="executor-<executor name>",
+ description = "Optionally specify the Buildx builder used to build docker images. " +
+ "Leave empty to use <code>executor-&lt;executor name&gt;</code>. OneDev will create the builder " +
+ "automatically if it does not exist. Jobs using the same builder can read or modify each other's " +
+ "build cache, so only share a builder among trusted jobs. Jobs using this executor still share its builder. Check " +
"<a href='https://docs.onedev.io/tutorials/cicd/insecure-docker-registry' target='_blank'>this tutorial</a> " +
"on how to customize the builder for instance to allow publishing to insecure registries")
- @NotEmpty
public String getDockerBuilder() {
return dockerBuilder;
}
@@ -186,6 +188,10 @@ public class ServerDockerExecutor extends JobExecutor implements DockerAware, Te
public void setDockerBuilder(String dockerBuilder) {
this.dockerBuilder = dockerBuilder;
}
+
+ protected String getEffectiveDockerBuilder() {
+ return getDockerBuilder() != null ? getDockerBuilder() : "executor-" + getName();
+ }
@Editable(order=600, group="Security Settings", description = "Whether or not to always pull image when " +
"run container or build images. This option should be enabled to avoid images being replaced by " +
@@ -495,7 +501,7 @@ public class ServerDockerExecutor extends JobExecutor implements DockerAware, Te
var docker = newDocker();
var registryLogins = merge(buildImageFacade.getRegistryLogins(), getRegistryLogins(jobToken));
callWithRegistryLogins(docker, registryLogins, () -> {
- JobUtils.buildImage(docker, getDockerBuilder(), getBuildOptions(), buildImageFacade, buildDir,
+ JobUtils.buildImage(docker, getEffectiveDockerBuilder(), getBuildOptions(), buildImageFacade, buildDir,
isAlwaysPullImage(), jobLogger);
return null;
});
@@ -511,7 +517,7 @@ public class ServerDockerExecutor extends JobExecutor implements DockerAware, Te
var pruneBuilderCacheFacade = (PruneBuilderCacheFacade) facade;
var docker = newDocker();
callWithRegistryLogins(docker, new ArrayList<>(), () -> {
- JobUtils.pruneBuilderCache(docker, getDockerBuilder(), pruneBuilderCacheFacade,
+ JobUtils.pruneBuilderCache(docker, getEffectiveDockerBuilder(), pruneBuilderCacheFacade,
buildDir, jobLogger);
return null;
});
diff --git a/server-product/system/incompatibilities/incompatibilities.md b/server-product/system/incompatibilities/incompatibilities.md
index f77fabbca5..3e305187c6 100644
--- a/server-product/system/incompatibilities/incompatibilities.md
+++ b/server-product/system/incompatibilities/incompatibilities.md
@@ -6,6 +6,10 @@ The `More Options` setting (`moreOptions` in build specs) of the build image ste
Existing step options are removed when build specs are migrated and are not copied to executors automatically. Before upgrading, record any options you need to retain and ask an administrator to configure them on the appropriate Docker job executor after upgrading. Executor options apply to all build image steps using that executor.
+The `Buildx Builder` setting of all server and remote Docker job executors is cleared during upgrade, including custom builder names. When unset, it now defaults to `executor-<executor name>` instead of `onedev`. Image builds and cache pruning use this executor-specific builder, so existing builder caches and custom configuration are no longer used by default. Before upgrading, record any custom builder settings and reconfigure the new builders or explicitly restore a builder name afterward if needed.
+
+Jobs using the same Buildx builder can read or modify each other's build cache. Only share a builder among trusted jobs; jobs using the same executor still share its builder. Use separate executors and builders for mutually untrusted jobs.
+
# 16.8.0
### CI/CD