The Unified and Autonomous Development Platform
Fix the issue that docker created files can not be deleted from host
Robin Shen9480af402043
diff --git a/core/src/main/java/io/onedev/server/ci/job/DefaultJobScheduler.java b/core/src/main/java/io/onedev/server/ci/job/DefaultJobScheduler.javaindex 018b3f9912..b3406bebf9 100644--- a/core/src/main/java/io/onedev/server/ci/job/DefaultJobScheduler.java+++ b/core/src/main/java/io/onedev/server/ci/job/DefaultJobScheduler.java@@ -340,6 +340,7 @@ public class DefaultJobScheduler implements JobScheduler, Runnable, SchedulableTthrow e;} finally {logger.info("Deleting workspace...");+ executor.cleanDir(workspace);FileUtils.deleteDir(workspace);logger.info("Workspace deleted");}diff --git a/core/src/main/java/io/onedev/server/model/support/jobexecutor/JobExecutor.java b/core/src/main/java/io/onedev/server/model/support/jobexecutor/JobExecutor.javaindex a136108a16..63e38bf5e2 100644--- a/core/src/main/java/io/onedev/server/model/support/jobexecutor/JobExecutor.java+++ b/core/src/main/java/io/onedev/server/model/support/jobexecutor/JobExecutor.java@@ -169,4 +169,6 @@ public abstract class JobExecutor implements Serializable {public abstract void checkCaches();+ public abstract void cleanDir(File dir);+}diff --git a/core/src/main/java/io/onedev/server/model/support/jobexecutor/KubernetesExecutor.java b/core/src/main/java/io/onedev/server/model/support/jobexecutor/KubernetesExecutor.javaindex cd0154ed3d..aa7d7b2820 100644--- a/core/src/main/java/io/onedev/server/model/support/jobexecutor/KubernetesExecutor.java+++ b/core/src/main/java/io/onedev/server/model/support/jobexecutor/KubernetesExecutor.java@@ -28,4 +28,9 @@ public class KubernetesExecutor extends JobExecutor {throw new UnsupportedOperationException();}+ @Override+ public void cleanDir(File dir) {+ throw new UnsupportedOperationException();+ }+}\ No newline at end of filediff --git a/core/src/main/java/io/onedev/server/model/support/jobexecutor/ServerDockerExecutor.java b/core/src/main/java/io/onedev/server/model/support/jobexecutor/ServerDockerExecutor.javaindex 618c1eadb3..c5c824c195 100644--- a/core/src/main/java/io/onedev/server/model/support/jobexecutor/ServerDockerExecutor.java+++ b/core/src/main/java/io/onedev/server/model/support/jobexecutor/ServerDockerExecutor.java@@ -14,6 +14,7 @@ import java.util.concurrent.Callable;import javax.validation.ConstraintValidatorContext;import org.apache.commons.codec.Charsets;+import org.apache.commons.lang3.SystemUtils;import org.hibernate.validator.constraints.NotEmpty;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -66,7 +67,9 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDaprivate String runOptions;private int capacity = Runtime.getRuntime().availableProcessors();-++ private String maintenanceImage = "busybox";+private transient ConstrainedRunner constrainedRunner;@Editable(order=1000, description="Optionally specify docker executable, for instance <i>/usr/local/bin/docker</i>. "@@ -143,7 +146,19 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDapublic void setCapacity(int capacity) {this.capacity = capacity;}-++ @Editable(order=22000, group="More Settings", description="A maintenance docker image is used by OneDev to do things requiring root "+ + "privilege. For instance, job execution in container may generate files owned by root user in cache "+ + "directory on host, and OneDev will run this maintenance image to remove outdated cache files as root if necessary")+ @NotEmpty+ public String getMaintenanceImage() {+ return maintenanceImage;+ }++ public void setMaintenanceImage(String maintenanceImage) {+ this.maintenanceImage = maintenanceImage;+ }+private Commandline getDockerCmd() {if (getDockerExecutable() != null)return new Commandline(getDockerExecutable());@@ -286,7 +301,7 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDathrow new RuntimeException(e);}cmd.addArgs(environment);- cmd.addArgs("sh", "-c", dockerWorkspacePath + "/onedev-job-commands.sh");+ cmd.addArgs("sh", dockerWorkspacePath + "/onedev-job-commands.sh");}logger.info("Running container to execute job...");@@ -391,21 +406,11 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDaFile lockFile = new File(cacheInstance, JobCache.LOCK_FILE);try {if (lockFile.createNewFile()) {- /*- * Remove other files first to avoid locking for too long time- */- for (File each: cacheInstance.listFiles()) {- if (!each.getName().equals(JobCache.LOCK_FILE)) {- if (each.isFile())- FileUtils.deleteFile(each);- else- FileUtils.deleteDir(each);- }- }LockUtils.call(keyDir.getAbsolutePath(), new Callable<Void>() {@Overridepublic Void call() throws Exception {+ cleanDir(cacheInstance);FileUtils.deleteDir(cacheInstance);return null;}@@ -467,25 +472,25 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDacmd.addArgs("run", "--rm");if (getRunOptions() != null)cmd.addArgs(StringUtils.parseQuoteTokens(getRunOptions()));- String dockerWorkspacePath;- String dockerCachePath = "$onedev-cache-test$";+ String containerWorkspacePath;+ String containerCachePath = "$onedev-cache-test$";if (windows) {- dockerWorkspacePath = "C:\\" + WORKSPACE;- dockerCachePath = "C:\\" + dockerCachePath;+ containerWorkspacePath = "C:\\" + WORKSPACE;+ containerCachePath = "C:\\" + containerCachePath;} else {- dockerWorkspacePath = "/" + WORKSPACE;- dockerCachePath = "/" + dockerCachePath;+ containerWorkspacePath = "/" + WORKSPACE;+ containerCachePath = "/" + containerCachePath;}- cmd.addArgs("-v", workspaceDir.getAbsolutePath() + ":" + dockerWorkspacePath);- cmd.addArgs("-v", cacheDir.getAbsolutePath() + ":" + dockerCachePath);+ cmd.addArgs("-v", workspaceDir.getAbsolutePath() + ":" + containerWorkspacePath);+ cmd.addArgs("-v", cacheDir.getAbsolutePath() + ":" + containerCachePath);- cmd.addArgs("-w", dockerWorkspacePath);- cmd.addArgs(testData.getDockerImage());+ cmd.addArgs("-w", containerWorkspacePath);+ cmd.addArgs(getPullImage(testData.getDockerImage()));if (windows)- cmd.addArgs("cmd", "/c", "echo this is a test");+ cmd.addArgs("cmd", "/c", "echo hello from container");else- cmd.addArgs("sh", "-c", "echo this is a test");+ cmd.addArgs("sh", "-c", "echo hello from container");cmd.execute(newInfoLogger(logger), newErrorLogger(logger)).checkReturnCode();} finally {@@ -496,8 +501,28 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDaif (!cacheHomeExists)FileUtils.deleteDir(cacheHome);}++ if (!SystemUtils.IS_OS_WINDOWS) {+ logger.info("Running maintenance container...");+ cmd = getDockerCmd();+ cmd.addArgs("run", "--rm", getPullImage(getMaintenanceImage()), "sh", "-c", "echo hello from maintenance container");+ cmd.execute(newInfoLogger(logger), newErrorLogger(logger)).checkReturnCode();+ }}+ @Override+ public void cleanDir(File dir) {+ if (SystemUtils.IS_OS_WINDOWS) {+ FileUtils.cleanDir(dir);+ } else {+ Commandline cmd = getDockerCmd();+ String containerPath = "/onedev_dir_to_clean";+ cmd.addArgs("run", "-v", dir.getAbsolutePath() + ":" + containerPath, "--rm",+ getPullImage(getMaintenanceImage()), "sh", "-c", "rm -rf " + containerPath + "/*");+ cmd.execute(newInfoLogger(logger), newErrorLogger(logger)).checkReturnCode();+ }+ }+@Editable(name="Specify a Docker Image to Test Against")public static class TestData implements Serializable {diff --git a/core/src/main/java/io/onedev/server/web/page/admin/jobexecutor/JobExecutorEditPanel.java b/core/src/main/java/io/onedev/server/web/page/admin/jobexecutor/JobExecutorEditPanel.javaindex f85a7a2e18..c64c12e1b4 100644--- a/core/src/main/java/io/onedev/server/web/page/admin/jobexecutor/JobExecutorEditPanel.java+++ b/core/src/main/java/io/onedev/server/web/page/admin/jobexecutor/JobExecutorEditPanel.java@@ -3,10 +3,13 @@ package io.onedev.server.web.page.admin.jobexecutor;import java.io.Serializable;import java.lang.reflect.ParameterizedType;import java.lang.reflect.Type;+import java.util.HashSet;import java.util.List;+import java.util.Set;import javax.annotation.Nullable;+import org.apache.commons.lang.SystemUtils;import org.apache.wicket.ajax.AjaxRequestTarget;import org.apache.wicket.ajax.markup.html.AjaxLink;import org.apache.wicket.ajax.markup.html.form.AjaxButton;@@ -69,7 +72,11 @@ abstract class JobExecutorEditPanel extends Panel {if (executorIndex != -1)bean.setExecutor(executors.get(executorIndex));- BeanEditor editor = BeanContext.editBean("editor", bean);+ Set<String> excludedProperties = new HashSet<>();+ if (SystemUtils.IS_OS_WINDOWS)+ excludedProperties.add("maintenanceImage");++ BeanEditor editor = BeanContext.editBean("editor", bean, excludedProperties, true);editor.setOutputMarkupId(true);AjaxButton saveButton = new AjaxButton("save") {diff --git a/core/src/main/java/io/onedev/server/web/page/admin/jobexecutor/JobExecutorPanel.java b/core/src/main/java/io/onedev/server/web/page/admin/jobexecutor/JobExecutorPanel.javaindex 36516d821a..b747025267 100644--- a/core/src/main/java/io/onedev/server/web/page/admin/jobexecutor/JobExecutorPanel.java+++ b/core/src/main/java/io/onedev/server/web/page/admin/jobexecutor/JobExecutorPanel.java@@ -1,7 +1,10 @@package io.onedev.server.web.page.admin.jobexecutor;+import java.util.HashSet;import java.util.List;+import java.util.Set;+import org.apache.commons.lang.SystemUtils;import org.apache.wicket.Component;import org.apache.wicket.ajax.AjaxRequestTarget;import org.apache.wicket.ajax.attributes.AjaxRequestAttributes;@@ -100,7 +103,11 @@ abstract class JobExecutorPanel extends Panel {});- add(BeanContext.viewBean("executor", getExecutor()).setOutputMarkupId(true));+ Set<String> excludedProperties = new HashSet<>();+ if (SystemUtils.IS_OS_WINDOWS)+ excludedProperties.add("maintenanceImage");++ add(BeanContext.viewBean("executor", getExecutor(), excludedProperties, true).setOutputMarkupId(true));add(AttributeAppender.append("class", new LoadableDetachableModel<String>() {