The Unified and Autonomous Development Platform
chore: Add workspace docker provision module
Robin Shend986e24d1e1b
diff --git a/pom.xml b/pom.xmlindex b22a3b45d2..0264b88803 100644--- a/pom.xml+++ b/pom.xml@@ -677,7 +677,7 @@</repositories><properties><commons.version>3.2.9</commons.version>- <agent.version>2.5.2</agent.version>+ <agent.version>2.5.3</agent.version><slf4j.version>2.0.9</slf4j.version><logback.version>1.4.14</logback.version><antlr.version>4.7.2</antlr.version>diff --git a/server-core/src/main/java/io/onedev/server/model/support/administration/GroovyScript.java b/server-core/src/main/java/io/onedev/server/model/support/administration/GroovyScript.javaindex a10f052570..b7a92309f6 100644--- a/server-core/src/main/java/io/onedev/server/model/support/administration/GroovyScript.java+++ b/server-core/src/main/java/io/onedev/server/model/support/administration/GroovyScript.java@@ -93,7 +93,7 @@ public class GroovyScript implements Serializable {"Use '**', '*' or '?' for <a href='https://docs.onedev.io/appendix/path-wildcard' target='_blank'>path wildcard match</a>. " +"Multiple projects should be separated by space")@Patterns(suggester="suggestProjects", path=true)- @DependsOn(property="canBeUsedByWorkspaces")+ @DependsOn(property="canBeUsedByWorkspaceSpecs")public String getApplicableWorkspaceProjects() {return applicableWorkspaceProjects;}diff --git a/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/ClaudeCodeInContainer.java b/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/ClaudeCodeInContainer.javaindex 14a08a5fc4..4260d45bc6 100644--- a/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/ClaudeCodeInContainer.java+++ b/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/ClaudeCodeInContainer.java@@ -10,12 +10,10 @@ import io.onedev.server.model.support.workspace.spec.WorkspaceSpec;import io.onedev.server.model.support.workspace.spec.shell.CustomLinuxShell;import io.onedev.server.web.util.SuggestionUtils;-@Editable(order=1100, name="nt:Claude Code in Container", description="""+@Editable(order=200, name="nt:Claude Code in Container", description="""Create a workspace spec running Claude Code inside container for isolation and security purpose.You may customize the <a href='https://code.onedev.io/onedev/docker/claudecode' target='_blank'>container image</a>- later to suit your needs if desired.- <b class='text-info'>NOTE: </b> Creating workspace inside container requires an enterprise subscription.- <a href='https://onedev.io/pricing' target='_blank'>Try free</a> for 30 days""")+ later to suit your needs if desired""")public class ClaudeCodeInContainer extends WorkspaceSpecTemplate {@Overridediff --git a/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/ClaudeCode.java b/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/ClaudeCodeInShell.javasimilarity index 83%rename from server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/ClaudeCode.javarename to server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/ClaudeCodeInShell.javaindex bc166b9970..0c8d85db28 100644--- a/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/ClaudeCode.java+++ b/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/ClaudeCodeInShell.java@@ -8,14 +8,15 @@ import io.onedev.server.model.support.workspace.spec.ShortcutConfig;import io.onedev.server.model.support.workspace.spec.WorkspaceSpec;import io.onedev.server.web.util.SuggestionUtils;-@Editable(order=200, name="nt:Claude Code", description="""+@Editable(order=1200, name="nt:Claude Code In Shell", description="""Create a workspace spec running Claude Code directly with server shell environment.- To use spec created by this template, please ensure that:+ Workspaces using created spec will share same environments and settings.+ To use this template, please ensure that:<ul><li><a href='https://code.claude.com/docs/en/overview' target='_blank'>Claude Code</a> is installed on OneDev server</li><li>A shell provisioner is added in 'Administration / Workspace Provisioners' with 'applicable projects' configured properly</li></ul>""")-public class ClaudeCode extends WorkspaceSpecTemplate {+public class ClaudeCodeInShell extends WorkspaceSpecTemplate {@Overridepublic WorkspaceSpec createWorkspaceSpec() {diff --git a/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/CodexInContainer.java b/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/CodexInContainer.javaindex 45355d6549..d30f1455de 100644--- a/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/CodexInContainer.java+++ b/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/CodexInContainer.java@@ -10,12 +10,10 @@ import io.onedev.server.model.support.workspace.spec.WorkspaceSpec;import io.onedev.server.model.support.workspace.spec.shell.CustomLinuxShell;import io.onedev.server.web.util.SuggestionUtils;-@Editable(order=1100, name="nt:Codex in Container", description="""+@Editable(order=300, name="nt:Codex in Container", description="""Create a workspace spec running Codex inside container for isolation and security purpose.You may customize the <a href='https://code.onedev.io/onedev/docker/codex' target='_blank'>container image</a>- later to suit your needs if desired.- <b class='text-info'>NOTE: </b> Creating workspace inside container requires an enterprise subscription.- <a href='https://onedev.io/pricing' target='_blank'>Try free</a> for 30 days""")+ later to suit your needs if desired""")public class CodexInContainer extends WorkspaceSpecTemplate {@Overridediff --git a/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/Codex.java b/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/CodexInShell.javasimilarity index 83%rename from server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/Codex.javarename to server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/CodexInShell.javaindex f69b8f57d4..6d68bb2593 100644--- a/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/Codex.java+++ b/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/CodexInShell.java@@ -8,14 +8,15 @@ import io.onedev.server.model.support.workspace.spec.ShortcutConfig;import io.onedev.server.model.support.workspace.spec.WorkspaceSpec;import io.onedev.server.web.util.SuggestionUtils;-@Editable(order=300, name="nt:Codex", description="""+@Editable(order=1300, name="nt:Codex In Shell", description="""Create a workspace spec running Codex directly with server shell environment.- To use spec created by this template, please ensure that:+ Workspaces using created spec will share same environments and settings.+ To use this template, please ensure that:<ul><li><a href='https://openai.com/codex/' target='_blank'>Codex</a> is installed on OneDev server</li><li>A shell provisioner is added in 'Administration / Workspace Provisioners' with 'applicable projects' configured properly</li></ul>""")-public class Codex extends WorkspaceSpecTemplate {+public class CodexInShell extends WorkspaceSpecTemplate {@Overridepublic WorkspaceSpec createWorkspaceSpec() {diff --git a/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/OpenCodeInContainer.java b/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/OpenCodeInContainer.javaindex 42b0fa628f..2e11f6de33 100644--- a/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/OpenCodeInContainer.java+++ b/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/OpenCodeInContainer.java@@ -10,12 +10,10 @@ import io.onedev.server.model.support.workspace.spec.WorkspaceSpec;import io.onedev.server.model.support.workspace.spec.shell.CustomLinuxShell;import io.onedev.server.web.util.SuggestionUtils;-@Editable(order=1000, name="nt:Open Code in Container", description="""+@Editable(order=100, name="nt:Open Code in Container", description="""Create a workspace spec running Open Code inside container for isolation and security purpose.You may customize the <a href='https://code.onedev.io/onedev/docker/opencode' target='_blank'>container image</a>- later to suit your needs if desired.- <b class='text-info'>NOTE: </b> Creating workspace inside container requires an enterprise subscription.- <a href='https://onedev.io/pricing' target='_blank'>Try free</a> for 30 days""")+ later to suit your needs if desired""")public class OpenCodeInContainer extends WorkspaceSpecTemplate {@Overridediff --git a/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/OpenCode.java b/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/OpenCodeInShell.javasimilarity index 83%rename from server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/OpenCode.javarename to server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/OpenCodeInShell.javaindex 0c35bbe176..b41aa03dba 100644--- a/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/OpenCode.java+++ b/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/template/OpenCodeInShell.java@@ -8,14 +8,15 @@ import io.onedev.server.model.support.workspace.spec.ShortcutConfig;import io.onedev.server.model.support.workspace.spec.WorkspaceSpec;import io.onedev.server.web.util.SuggestionUtils;-@Editable(order=100, name="nt:Open Code", description="""+@Editable(order=1100, name="nt:Open Code In Shell", description="""Create a workspace spec running Open Code directly with server shell environment.- To use spec created by this template, please ensure that:+ Workspaces using created spec will share same environments and settings.+ To use this template, please ensure that:<ul><li><a href='https://opencode.ai/' target='_blank'>Open Code</a> is installed on OneDev server</li><li>A shell provisioner is added in 'Administration / Workspace Provisioners' with 'applicable projects' configured properly</li></ul>""")-public class OpenCode extends WorkspaceSpecTemplate {+public class OpenCodeInShell extends WorkspaceSpecTemplate {@Overridepublic WorkspaceSpec createWorkspaceSpec() {diff --git a/server-ee b/server-eeindex 154cb834d8..cc31b3d077 160000--- a/server-ee+++ b/server-ee@@ -1 +1 @@-Subproject commit 154cb834d869fb4c5e90b84186bfcc6f7e8c3318+Subproject commit cc31b3d077f3e51c782f8ccee5febc1fdb624b50diff --git a/server-plugin/pom.xml b/server-plugin/pom.xmlindex 440aa66e2e..fb0bf29e2c 100644--- a/server-plugin/pom.xml+++ b/server-plugin/pom.xml@@ -44,6 +44,7 @@<module>server-plugin-executor-servershell</module><module>server-plugin-executor-remoteshell</module><module>server-plugin-provisioner-shell</module>+ <module>server-plugin-provisioner-docker</module><module>server-plugin-report-unittest</module><module>server-plugin-report-junit</module><module>server-plugin-report-gtest</module>diff --git a/server-plugin/server-plugin-provisioner-docker/pom.xml b/server-plugin/server-plugin-provisioner-docker/pom.xmlnew file mode 100644index 0000000000..802f7c39ce--- /dev/null+++ b/server-plugin/server-plugin-provisioner-docker/pom.xml@@ -0,0 +1,13 @@+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">+ <modelVersion>4.0.0</modelVersion>+ <artifactId>server-plugin-provisioner-docker</artifactId>+ <parent>+ <groupId>io.onedev</groupId>+ <artifactId>server-plugin</artifactId>+ <version>15.0.0</version>+ </parent>+ <properties>+ <moduleClass>io.onedev.server.plugin.provisioner.docker.DockerModule</moduleClass>+ </properties>+</project>diff --git a/server-plugin/server-plugin-provisioner-docker/src/main/java/io/onedev/server/plugin/provisioner/docker/ConfigFileProvisioner.java b/server-plugin/server-plugin-provisioner-docker/src/main/java/io/onedev/server/plugin/provisioner/docker/ConfigFileProvisioner.javanew file mode 100644index 0000000000..bbd5b9c684--- /dev/null+++ b/server-plugin/server-plugin-provisioner-docker/src/main/java/io/onedev/server/plugin/provisioner/docker/ConfigFileProvisioner.java@@ -0,0 +1,50 @@+package io.onedev.server.plugin.provisioner.docker;++import java.io.File;+import java.io.IOException;+import java.nio.charset.StandardCharsets;+import java.util.HashMap;+import java.util.List;+import java.util.Map;++import io.onedev.commons.utils.ExplicitException;+import io.onedev.commons.utils.FileUtils;+import io.onedev.server.annotation.Path;+import io.onedev.server.model.support.workspace.spec.ConfigFile;+import io.onedev.server.validation.validator.PathValidator;+import io.onedev.server.workspace.WorkspaceContext;++public class ConfigFileProvisioner {++ private final File workspaceDir;++ private final List<ConfigFile> configFiles;++ public ConfigFileProvisioner(File workspaceDir, WorkspaceContext context) {+ this.workspaceDir = workspaceDir;+ this.configFiles = context.getSpec().getConfigFiles();+ }++ public Map<String, File> setup() {+ var cacheIndex = 1;+ var pathMap = new HashMap<String, File>();+ for (var configFile : configFiles) {+ var path = configFile.getPath();+ String checkMessage = PathValidator.checkPath(Path.Type.ABSOLUTE, path);+ if (checkMessage != null)+ throw new ExplicitException("Invalid config file path '" + path + "': " + checkMessage);++ File pathFile = new File(workspaceDir, "config-files/" + (cacheIndex++));++ try {+ FileUtils.writeStringToFile(pathFile, configFile.getContent(), StandardCharsets.UTF_8);+ } catch (IOException e) {+ throw new RuntimeException(e);+ }++ pathMap.put(path, pathFile);+ }+ return pathMap;+ }++}diff --git a/server-plugin/server-plugin-provisioner-docker/src/main/java/io/onedev/server/plugin/provisioner/docker/DockerModule.java b/server-plugin/server-plugin-provisioner-docker/src/main/java/io/onedev/server/plugin/provisioner/docker/DockerModule.javanew file mode 100644index 0000000000..f11efb01ba--- /dev/null+++ b/server-plugin/server-plugin-provisioner-docker/src/main/java/io/onedev/server/plugin/provisioner/docker/DockerModule.java@@ -0,0 +1,81 @@+package io.onedev.server.plugin.provisioner.docker;++import java.util.Collection;++import com.google.common.collect.Sets;++import io.onedev.agent.AgentUtils;+import io.onedev.commons.loader.AbstractPluginModule;+import io.onedev.commons.loader.ImplementationProvider;+import io.onedev.commons.utils.command.Commandline;+import io.onedev.commons.utils.command.LineConsumer;+import io.onedev.server.OneDev;+import io.onedev.server.exception.ExceptionUtils;+import io.onedev.server.model.support.administration.workspaceprovisioner.WorkspaceProvisioner;+import io.onedev.server.workspace.WorkspaceProvisionerDiscoverer;++/**+ * NOTE: Do not forget to rename moduleClass property defined in the pom if you've renamed this class.+ *+ */+public class DockerModule extends AbstractPluginModule {++ @Override+ protected void configure() {+ super.configure();++ if (OneDev.getK8sService() == null) {+ contribute(ImplementationProvider.class, new ImplementationProvider() {++ @Override+ public Class<?> getAbstractClass() {+ return WorkspaceProvisioner.class;+ }++ @Override+ public Collection<Class<?>> getImplementations() {+ return Sets.newHashSet(DockerProvisioner.class);+ }++ });++ contribute(WorkspaceProvisionerDiscoverer.class, new WorkspaceProvisionerDiscoverer() {++ @Override+ public WorkspaceProvisioner discover() {+ var docker = new Commandline(AgentUtils.getDockerExecutable(null));+ docker.addArgs("version");+ try {+ docker.execute(new LineConsumer() {++ @Override+ public void consume(String line) {+ }++ }, new LineConsumer() {++ @Override+ public void consume(String line) {+ }++ }).checkReturnCode();++ return new DockerProvisioner();+ } catch (Exception e) {+ if (ExceptionUtils.find(e, InterruptedException.class) != null)+ throw ExceptionUtils.unchecked(e);+ else+ return null;+ }+ }++ @Override+ public int getOrder() {+ return DockerProvisioner.ORDER;+ }++ });+ }+ }++}diff --git a/server-plugin/server-plugin-provisioner-docker/src/main/java/io/onedev/server/plugin/provisioner/docker/DockerProvisioner.java b/server-plugin/server-plugin-provisioner-docker/src/main/java/io/onedev/server/plugin/provisioner/docker/DockerProvisioner.javanew file mode 100644index 0000000000..96b1465f81--- /dev/null+++ b/server-plugin/server-plugin-provisioner-docker/src/main/java/io/onedev/server/plugin/provisioner/docker/DockerProvisioner.java@@ -0,0 +1,601 @@+package io.onedev.server.plugin.provisioner.docker;++import static io.onedev.agent.AgentUtils.callWithRegistryLogins;+import static io.onedev.agent.AgentUtils.changeOwner;+import static io.onedev.agent.AgentUtils.getOsIds;+import static io.onedev.agent.AgentUtils.newDockerKiller;+import static io.onedev.k8shelper.RegistryLoginFacade.merge;+import static java.util.stream.Collectors.toList;++import java.io.ByteArrayOutputStream;+import java.io.File;+import java.io.IOException;+import java.io.Serializable;+import java.nio.charset.StandardCharsets;+import java.util.ArrayList;+import java.util.Collection;+import java.util.Date;+import java.util.HashMap;+import java.util.HashSet;+import java.util.List;+import java.util.Map;+import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.ExecutionException;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.ThreadLocalRandom;+import java.util.concurrent.atomic.AtomicBoolean;++import javax.validation.Valid;+import javax.validation.constraints.NotEmpty;++import org.apache.commons.io.FilenameUtils;+import org.apache.commons.lang3.SystemUtils;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++import com.google.common.base.Preconditions;++import io.onedev.agent.AgentUtils;+import io.onedev.commons.bootstrap.Bootstrap;+import io.onedev.commons.utils.ExplicitException;+import io.onedev.commons.utils.FileUtils;+import io.onedev.commons.utils.StringUtils;+import io.onedev.commons.utils.TaskLogger;+import io.onedev.commons.utils.command.Commandline;+import io.onedev.commons.utils.command.LineConsumer;+import io.onedev.k8shelper.KubernetesHelper;+import io.onedev.k8shelper.RegistryLoginFacade;+import io.onedev.server.OneDev;+import io.onedev.server.annotation.Editable;+import io.onedev.server.annotation.OmitName;+import io.onedev.server.annotation.ReservedOptions;+import io.onedev.server.cache.WorkspaceCacheProvisioner;+import io.onedev.server.model.support.administration.DockerAware;+import io.onedev.server.model.support.administration.workspaceprovisioner.RegistryLogin;+import io.onedev.server.model.support.administration.workspaceprovisioner.WorkspaceProvisioner;+import io.onedev.server.terminal.CommandlineShell;+import io.onedev.server.terminal.Shell;+import io.onedev.server.terminal.Terminal;+import io.onedev.server.web.util.Testable;+import io.onedev.server.workspace.GitExecutionResult;+import io.onedev.server.workspace.WorkspaceContext;+import io.onedev.server.workspace.WorkspaceRuntime;++@Editable(order=DockerProvisioner.ORDER, name="Docker Provisioner", description = """+ This provisioner creates workspaces inside Docker containers on the OneDev server for+ security and isolation purpose. It is currently not supported when OneDev server is+ installed inside Kubernetes cluster""")+public class DockerProvisioner extends WorkspaceProvisioner implements DockerAware, Testable<DockerProvisioner.TestData> {++ private static final long serialVersionUID = 1L;++ private static final Logger logger = LoggerFactory.getLogger(DockerProvisioner.class);++ public static final int ORDER = 100;++ private static final int MAX_PORT_CONFLICT_RETRIES = 5;++ private static final String CONTAINER_WORKSPACE_PATH = "/onedev-workspace";++ private List<RegistryLogin> registryLogins = new ArrayList<>();++ private String runOptions;++ private boolean alwaysPullImage = true;++ private String dockerExecutable;++ private boolean mountDockerSock;++ private String dockerSockPath;++ private String cpuLimit;++ private String memoryLimit;++ private static volatile String hostInstallPath;++ @Editable(order=400, description="Specify registry logins if necessary. For built-in registry, " ++ "use <code>@server_url@</code> for registry url, <code>@job_token@</code> for user name, and " ++ "access token for password")+ @Valid+ public List<RegistryLogin> getRegistryLogins() {+ return registryLogins;+ }++ public void setRegistryLogins(List<RegistryLogin> registryLogins) {+ this.registryLogins = registryLogins;+ }++ @Editable(order=40, group="Resource Settings", placeholder = "No limit", description = "" ++ "Optionally specify cpu limit for workspace containers. This will be " ++ "used as option <a href='https://docs.docker.com/config/containers/resource_constraints/#cpu' target='_blank'>--cpus</a> " ++ "of the container")+ public String getCpuLimit() {+ return cpuLimit;+ }++ public void setCpuLimit(String cpuLimit) {+ this.cpuLimit = cpuLimit;+ }++ @Editable(order=50, group="Resource Settings", placeholder = "No limit", description = "" ++ "Optionally specify memory limit for workspace containers. This will be " ++ "used as option <a href='https://docs.docker.com/config/containers/resource_constraints/#memory' target='_blank'>--memory</a> " ++ "of the container")+ public String getMemoryLimit() {+ return memoryLimit;+ }++ public void setMemoryLimit(String memoryLimit) {+ this.memoryLimit = memoryLimit;+ }++ @Editable(order=30, group="Security Settings", description="Whether or not to mount docker sock into workspace container to "+ + "support docker operations in workspace<br>"+ + "<b class='text-danger'>WARNING</b>: Malicious workspaces can take control of whole OneDev "+ + "by operating the mounted docker sock. Make sure this provisioner can only be used by "+ + "trusted workspaces if this option is enabled")+ public boolean isMountDockerSock() {+ return mountDockerSock;+ }++ public void setMountDockerSock(boolean mountDockerSock) {+ this.mountDockerSock = mountDockerSock;+ }++ @Editable(order=600, group="Security Settings", description = "Whether or not to always pull image when "+ + "running container. This option should be enabled to avoid images being replaced by "+ + "malicious operations")+ public boolean isAlwaysPullImage() {+ return alwaysPullImage;+ }++ public void setAlwaysPullImage(boolean alwaysPullImage) {+ this.alwaysPullImage = alwaysPullImage;+ }++ @Editable(order=510, group="More Settings", placeholder="Default", description="Optionally specify docker sock to use. "+ + "Defaults to <i>/var/run/docker.sock</i>")+ public String getDockerSockPath() {+ return dockerSockPath;+ }++ public void setDockerSockPath(String dockerSockPath) {+ this.dockerSockPath = dockerSockPath;+ }++ @Editable(order=50050, group="More Settings", description="Optionally specify docker options to run container. " ++ "Multiple options should be separated by space, and single option containing spaces should be quoted")+ @ReservedOptions({"-w", "(--workdir)=.*", "-d", "--detach", "-a", "--attach", "-t", "--tty",+ "-i", "--interactive", "--rm", "--restart", "(--name)=.*"})+ public String getRunOptions() {+ return runOptions;+ }++ public void setRunOptions(String runOptions) {+ this.runOptions = runOptions;+ }++ @Editable(order=50100, group="More Settings", placeholder="Use default", description=""+ + "Optionally specify docker executable, for instance <i>/usr/local/bin/docker</i>. "+ + "Leave empty to use docker executable in PATH")+ public String getDockerExecutable() {+ return dockerExecutable;+ }++ public void setDockerExecutable(String dockerExecutable) {+ this.dockerExecutable = dockerExecutable;+ }++ private Commandline newDocker() {+ var docker = new Commandline(AgentUtils.getDockerExecutable(getDockerExecutable()));+ AgentUtils.useDockerSock(docker, getDockerSockPath());+ return docker;+ }++ private String getHostPath(String path) {+ String installPath = Bootstrap.installDir.getAbsolutePath();+ Preconditions.checkState(path.startsWith(installPath + "/") || path.startsWith(installPath + "\\"));+ if (hostInstallPath == null) {+ if (Bootstrap.isInDocker())+ hostInstallPath = AgentUtils.getHostPath(newDocker(), installPath);+ else+ hostInstallPath = installPath;+ }+ return hostInstallPath + path.substring(installPath.length());+ }++ private List<RegistryLoginFacade> getRegistryLoginFacades(String token) {+ return getRegistryLogins().stream()+ .map(it -> it.getFacade(token))+ .collect(toList());+ }++ private void checkApplicable() {+ if (OneDev.getK8sService() != null) {+ throw new ExplicitException("OneDev running inside kubernetes cluster does not support workspaces yet");+ }+ }++ @Override+ public void test(TestData testData, TaskLogger jobLogger) {+ checkApplicable();+ var docker = newDocker();+ callWithRegistryLogins(docker, getRegistryLoginFacades(testData.getDockerImage()), () -> {+ jobLogger.log("Testing specified docker image...");+ docker.args("run", "--rm");+ if (getCpuLimit() != null)+ docker.addArgs("--cpus", getCpuLimit());+ if (getMemoryLimit() != null)+ docker.args("--memory", getMemoryLimit());+ if (getRunOptions() != null)+ docker.args(StringUtils.parseQuoteTokens(getRunOptions()));+ docker.args(testData.getDockerImage(), "sh", "-c", "echo hello from container");+ docker.execute(new LineConsumer() {+ @Override+ public void consume(String line) {+ jobLogger.log(line);+ }+ }, new LineConsumer() {+ @Override+ public void consume(String line) {+ jobLogger.log(line);+ }+ }).checkReturnCode();+ return null;+ });+ }++ @Editable(name="Specify a Docker Image to Test Against")+ public static class TestData implements Serializable {++ private static final long serialVersionUID = 1L;++ private String dockerImage;++ @Editable+ @OmitName+ @NotEmpty+ public String getDockerImage() {+ return dockerImage;+ }++ public void setDockerImage(String dockerImage) {+ this.dockerImage = dockerImage;+ }++ }++ private void deleteContainerIfExist(Commandline docker, String containerName, TaskLogger logger) {+ docker.args("rm", "-f", containerName);+ docker.execute(AgentUtils.newInfoLogger(logger), new LineConsumer() {+ @Override+ public void consume(String line) {+ if (!line.equals(containerName) && !line.contains("No such container"))+ logger.warning(line);+ }+ });+ }++ private void setCommonDockerRunOptions(Commandline docker, String containerName,+ String runAs, TaskLogger logger) {+ docker.args("run", "--rm", "--name=" + containerName);+ if (isAlwaysPullImage())+ docker.addArgs("--pull=always");++ docker.addArgs("--user", runAs);++ if (getCpuLimit() != null)+ docker.addArgs("--cpus", getCpuLimit());+ if (getMemoryLimit() != null)+ docker.addArgs("--memory", getMemoryLimit());++ docker.processKiller(newDockerKiller(docker, containerName, logger));+ }++ @Override+ public WorkspaceRuntime provision(WorkspaceContext context, TaskLogger logger) {+ checkApplicable();++ if (!context.getSpec().isRunInContainer())+ throw new ExplicitException("This workspace can only be provisioned by shell provisioner");++ var workspaceDir = getWorkspaceDir(context);+ var workDir = new File(workspaceDir, "work");+ FileUtils.createDir(workDir);++ var osIds = getOsIds(logger);+ if (SystemUtils.IS_OS_LINUX && !osIds.equals("0:0")) {+ logger.log("Changing owner of workspace directory to host user...");+ changeOwner(newDocker(), osIds, workspaceDir, osIds, logger);+ }++ if (!Bootstrap.isInDocker()) {+ var dockerExecutableFile = new File(workspaceDir, "docker-executable");+ FileUtils.writeFile(dockerExecutableFile, AgentUtils.getDockerExecutable(getDockerExecutable()));+ }++ var provisionerRegistryLogins = getRegistryLoginFacades(context.getToken());+ var registryLogins = context.getSpec().getRegistryLogins().stream()+ .map(it -> new RegistryLoginFacade(it.getRegistryUrl(), it.getUserName(), it.getPassword()))+ .collect(toList());+ var allRegistryLogins = merge(registryLogins, provisionerRegistryLogins);++ var infoLogger = AgentUtils.newInfoLogger(logger);+ var warningLogger = AgentUtils.newWarningLogger(logger);++ var envVars = setupRepository(context, CONTAINER_WORKSPACE_PATH, logger);++ envVars.put("TERM", "xterm-256color");+ envVars.put("LANG", "C.UTF-8");+ envVars.put("ONEDEV_WORKDIR", CONTAINER_WORKSPACE_PATH + "/work");++ logger.log("Setting up cache...");++ var cacheProvisioner = new WorkspaceCacheProvisioner(workspaceDir, context, logger);+ cacheProvisioner.setupCaches();++ logger.log("Setting up user data...");++ var userDataProvisioner = new UserDataProvisioner(context, workspaceDir, logger);+ var userDataPathMap = userDataProvisioner.setup();+ var userDataCheckDate = new Date();++ logger.log("Setting up config files...");++ var configFileProvisioner = new ConfigFileProvisioner(workspaceDir, context);+ var configFilePathMap = configFileProvisioner.setup();++ var containerWorkspacePath = CONTAINER_WORKSPACE_PATH;++ var entrypointArgs = new StringBuilder(KubernetesHelper.GIT_TRUST_ALL_DIRS);+ var shell = context.getSpec().getShell();+ var setupCommands = shell.getSetupCommands();+ if (setupCommands != null) {+ var scriptFile = new File(workspaceDir, "setup" + shell.getFacility().getScriptExtension());+ try {+ FileUtils.writeStringToFile(+ scriptFile,+ shell.getFacility().normalizeCommands(setupCommands),+ StandardCharsets.UTF_8);+ } catch (IOException e) {+ throw new RuntimeException(e);+ }++ var containerScriptPath = containerWorkspacePath + "/" + scriptFile.getName();+ entrypointArgs+ .append(" && cd /onedev-workspace/work && echo Running setup commands... && ")+ .append(shell.getFacility().getExecutable())+ .append(" ")+ .append(StringUtils.join(shell.getFacility().getScriptOptions(), " "))+ .append(" ")+ .append(containerScriptPath)+ .append(" && touch /onedev-workspace/successful && chmod +r /onedev-workspace/successful || exit 1");+ } else {+ entrypointArgs.append(" && touch /onedev-workspace/successful && chmod +r /onedev-workspace/successful");+ }++ entrypointArgs.append(" && tail -f /dev/null");++ var successfulFile = new File(workspaceDir, "successful");+ if (successfulFile.exists())+ FileUtils.deleteFile(successfulFile);++ var emptyUserDataMounts = new HashMap<String, File>();+ for (var userData: context.getSpec().getUserDatas()) {+ for (var path: userData.getPaths()) {+ var pathDir = Preconditions.checkNotNull(userDataPathMap.get(path));+ if (!pathDir.exists() || pathDir.list().length == 0) {+ FileUtils.createDir(pathDir);+ emptyUserDataMounts.put(path, pathDir);+ }+ }+ }++ var docker = newDocker();+ var runAs = context.getSpec().getRunAs();++ if (SystemUtils.IS_OS_LINUX && !runAs.equals("0:0")) {+ logger.log("Changing owner of workspace directory to container user...");+ changeOwner(docker, runAs, workspaceDir, osIds, logger);+ }++ if (!emptyUserDataMounts.isEmpty()) {+ logger.log("Initializing user data...");++ callWithRegistryLogins(docker, allRegistryLogins, () -> {+ var containerName = "workspace-" + getName() + "-" + context.getProjectId()+ + "-" + context.getWorkspaceNumber() + "-user-data-init";+ deleteContainerIfExist(docker, containerName, logger);++ setCommonDockerRunOptions(docker, containerName, runAs, logger);++ var initEntrypointArgs = new StringBuilder("set -e");+ for (var entry : emptyUserDataMounts.entrySet()) {+ var containerPath = entry.getKey();+ var mountedDirPath = "/onedev-user-data/" + entry.getValue().getName();+ docker.addArgs("-v", getHostPath(entry.getValue().getAbsolutePath()) + ":" + mountedDirPath);+ initEntrypointArgs.append(" ; ");+ initEntrypointArgs.append("if [ -f '").append(containerPath).append("' ]; then ")+ .append("cp '").append(containerPath).append("' '")+ .append(mountedDirPath).append("/").append(UserDataProvisioner.MOUNT_FILE).append("'")+ .append("; elif [ -d '").append(containerPath).append("' ]; then ")+ .append("cp -a '").append(containerPath).append("/.' '").append(mountedDirPath).append("/'")+ .append("; fi");+ }++ docker.addArgs("--entrypoint", "sh", context.getSpec().getImage(), "-c", initEntrypointArgs.toString());+ docker.execute(infoLogger, warningLogger).checkReturnCode();++ return null;+ });+ }++ var containerName = "workspace-" + getName() + "-" + context.getProjectId() + "-" + context.getWorkspaceNumber();+ var portMappings = new ConcurrentHashMap<Integer, Integer>();+ var future = OneDev.getInstance(ExecutorService.class).submit(() -> {+ logger.log("Starting docker container...");++ callWithRegistryLogins(docker, allRegistryLogins, () -> {+ deleteContainerIfExist(docker, containerName, logger);++ int attempt = 1;+ while (true) {+ setCommonDockerRunOptions(docker, containerName, runAs, logger);++ portMappings.clear();+ for (int containerPort : context.getSpec().getContainerPorts()) {+ int hostPort = ThreadLocalRandom.current().nextInt(10001, 65536);+ portMappings.put(containerPort, hostPort);+ docker.addArgs("-p", hostPort + ":" + containerPort);+ }++ docker.addArgs("-v", getHostPath(workspaceDir.getAbsolutePath()) + ":" + containerWorkspacePath);++ for (var allocation: cacheProvisioner.getAllocations()) {+ for (var entry: allocation.getPathMap().entrySet()) {+ if (FilenameUtils.getPrefixLength(entry.getKey()) > 0)+ docker.addArgs("-v", getHostPath(entry.getValue().getAbsolutePath()) + ":" + entry.getKey());+ }+ }++ for (var entry: userDataPathMap.entrySet()) {+ File mountFrom = entry.getValue();+ var file = new File(entry.getValue(), UserDataProvisioner.MOUNT_FILE);+ if (file.exists())+ mountFrom = file;+ docker.addArgs("-v", getHostPath(mountFrom.getAbsolutePath()) + "/:" + entry.getKey());+ }++ for (var entry: configFilePathMap.entrySet())+ docker.addArgs("-v", getHostPath(entry.getValue().getAbsolutePath()) + ":" + entry.getKey());++ if (isMountDockerSock()) {+ if (getDockerSockPath() != null)+ docker.addArgs("-v", getDockerSockPath() + ":/var/run/docker.sock");+ else+ docker.addArgs("-v", "/var/run/docker.sock:/var/run/docker.sock");+ }+ if (getRunOptions() != null)+ docker.addArgs(StringUtils.parseQuoteTokens(getRunOptions()));++ for (var entry : envVars.entrySet())+ docker.addArgs("-e", entry.getKey() + "=" + entry.getValue());++ docker.addArgs("--entrypoint", "sh", context.getSpec().getImage(), "-c", entrypointArgs.toString());++ AtomicBoolean portConflictDetected = new AtomicBoolean(false);++ var executeResult = docker.execute(infoLogger, new LineConsumer() {++ @Override+ public void consume(String line) {+ if (line.contains("port is already allocated") || line.contains("address already in use"))+ portConflictDetected.set(true);+ else if (!portConflictDetected.get() || !line.startsWith("See 'docker run --help'"))+ logger.warning(line);+ }++ });+ if (executeResult.getReturnCode() != 0) {+ if (!portConflictDetected.get() || attempt++ >= MAX_PORT_CONFLICT_RETRIES)+ throw new ExplicitException("Docker container exited with code " + executeResult.getReturnCode());+ else+ logger.log("Port conflict detected, retrying with different ports...");+ } else {+ break;+ }+ }+ return null;+ });+ });++ while (true) {+ if (future.isDone()) {+ try {+ future.get();+ } catch (InterruptedException|ExecutionException e) {+ throw new RuntimeException(e);+ }+ throw new ExplicitException("Docker container stopped unexpectedly");+ } else if (successfulFile.exists()) {+ break;+ }+ try {+ Thread.sleep(1000);+ } catch (InterruptedException e) {+ future.cancel(true);+ throw new RuntimeException(e);+ }+ }++ var containerWorkDirPath = containerWorkspacePath + "/work";+ return new WorkspaceRuntime() {++ @Override+ public GitExecutionResult executeGitCommand(String[] gitArgs) {+ var docker = newDocker();+ docker.addArgs("exec", "-w", containerWorkDirPath, containerName, "git");+ docker.addArgs(gitArgs);++ var stdoutStream = new ByteArrayOutputStream();+ var stderrStream = new ByteArrayOutputStream();+ var returnCode = docker.execute(stdoutStream, stderrStream).getReturnCode();+ return new GitExecutionResult(stdoutStream.toByteArray(), stderrStream.toByteArray(), returnCode);+ }++ @Override+ public Shell doOpenShell(Terminal terminal) {+ var docker = newDocker();+ docker.addArgs("exec", "-it", "--detach-keys=ctrl-z,z", "-w", containerWorkDirPath, containerName,+ "tmux", "new-session", context.getSpec().getShell().getFacility().getExecutable());+ return new CommandlineShell(terminal, docker);+ }++ @Override+ public void await() {+ try {+ future.get();+ } catch (InterruptedException e) {+ future.cancel(true);+ throw new RuntimeException(e);+ } catch (ExecutionException e) {+ throw new RuntimeException(e);+ } finally {+ if (SystemUtils.IS_OS_LINUX && !osIds.equals("0:0")) {+ var dirs = new HashSet<File>();+ cacheProvisioner.getAllocations().stream()+ .map(it->it.getPathMap().values())+ .flatMap(Collection::stream)+ .forEach(dirs::add);+ userDataPathMap.values().forEach(dirs::add);+ changeOwner(docker, osIds, dirs, osIds, logger);+ }+ try {+ cacheProvisioner.uploadCaches();+ } catch (Throwable t) {+ DockerProvisioner.logger.error("Error uploading caches", t);+ }+ try {+ userDataProvisioner.store(userDataCheckDate, userDataPathMap);+ } catch (Throwable t) {+ DockerProvisioner.logger.error("Error storing user data", t);+ }+ }+ }++ @Override+ public Map<Integer, Integer> getPortMappings() {+ return portMappings;+ }++ };+ }++ @Override+ public List<RegistryLoginFacade> getRegistryLogins(String token) {+ return getRegistryLoginFacades(token);+ }++}diff --git a/server-plugin/server-plugin-provisioner-docker/src/main/java/io/onedev/server/plugin/provisioner/docker/UserDataProvisioner.java b/server-plugin/server-plugin-provisioner-docker/src/main/java/io/onedev/server/plugin/provisioner/docker/UserDataProvisioner.javanew file mode 100644index 0000000000..37a9a2c05e--- /dev/null+++ b/server-plugin/server-plugin-provisioner-docker/src/main/java/io/onedev/server/plugin/provisioner/docker/UserDataProvisioner.java@@ -0,0 +1,182 @@+package io.onedev.server.plugin.provisioner.docker;++import static io.onedev.commons.utils.LockUtils.read;+import static io.onedev.commons.utils.LockUtils.write;++import java.io.File;+import java.io.FileInputStream;+import java.io.FileOutputStream;+import java.io.IOException;+import java.text.MessageFormat;+import java.util.Date;+import java.util.HashMap;+import java.util.List;+import java.util.Map;+import java.util.concurrent.atomic.AtomicInteger;++import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++import io.onedev.commons.bootstrap.Bootstrap;+import io.onedev.commons.utils.ExplicitException;+import io.onedev.commons.utils.FileUtils;+import io.onedev.commons.utils.TarUtils;+import io.onedev.commons.utils.TaskLogger;+import io.onedev.server.OneDev;+import io.onedev.server.annotation.Path;+import io.onedev.server.cluster.ClusterService;+import io.onedev.server.cluster.ClusterTask;+import io.onedev.server.model.User;+import io.onedev.server.model.support.workspace.spec.UserData;+import io.onedev.server.service.UserService;+import io.onedev.server.util.PathIndexUtils;+import io.onedev.server.util.SiteSyncUtils;+import io.onedev.server.validation.validator.PathValidator;+import io.onedev.server.workspace.WorkspaceContext;++public class UserDataProvisioner {++ private static final Logger logger = LoggerFactory.getLogger(UserDataProvisioner.class);++ static final String MOUNT_FILE = "onedev.this-should-be-mounted-instead";++ private final Long userId;++ private final List<UserData> userDatas;++ private final File workspaceDir;++ private final TaskLogger taskLogger;++ public UserDataProvisioner(WorkspaceContext context, File workspaceDir, TaskLogger taskLogger) {+ this.userId = context.getUserId();+ this.userDatas = context.getSpec().getUserDatas();+ this.workspaceDir = workspaceDir;+ this.taskLogger = taskLogger;+ }++ public Map<String, File> setup() {+ var pathMap = new HashMap<String, File>();+ var userService = getUserService();+ AtomicInteger cacheIndex = new AtomicInteger(1);+ for (var userData : userDatas) {+ var dataKey = userData.getKey();+ var dataStorageDir = userService.getWorkspaceDataDir(userId, dataKey, false);+ read(User.getWorkspaceDataLockName(userId, dataKey), () -> {+ try {+ var pathIndexes = PathIndexUtils.read(userService.getWorkspaceDataDir(userId, dataKey, false));+ for (var path: userData.getPaths()) {+ var pathCheckMessage = PathValidator.checkPath(Path.Type.ABSOLUTE, path);+ if (pathCheckMessage != null)+ throw new ExplicitException("Invalid user data path '" + path + "': " + pathCheckMessage);+ var pathDir = new File(workspaceDir, "user-data/" + cacheIndex.getAndIncrement());+ var storedIndex = pathIndexes.get(path);+ var pathStorageFile = storedIndex != null+ ? new File(dataStorageDir, String.valueOf(storedIndex))+ : null;+ if ((!pathDir.exists() || pathDir.list().length == 0)+ && pathStorageFile != null && pathStorageFile.exists()) {+ FileUtils.createDir(pathDir);+ try (var is = new FileInputStream(pathStorageFile)) {+ TarUtils.untar(is, pathDir, false);+ }+ }+ pathMap.put(path, pathDir);+ }+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ });+ }+ return pathMap;+ }++ private static File getUsersStorageDir() {+ return OneDev.getInstance(UserService.class).getUsersDir();+ }++ public void store(Date checkDate, Map<String, File> pathMap) {+ var usersStorageDir = getUsersStorageDir();+ var userService = getUserService();+ for (var userData: userDatas) {+ var dataKey = userData.getKey();+ var dataStorageDir = userService.getWorkspaceDataDir(userId, dataKey, true);+ var lockName = User.getWorkspaceDataLockName(userId, dataKey);+ for (var path: userData.getPaths()) {+ var pathDir = pathMap.get(path);+ if (pathDir != null && FileUtils.hasChangedFiles(pathDir, checkDate, userData.getChangeDetectionExcludes())) {+ taskLogger.log(MessageFormat.format("User data changed (key: {0}, path: {1}), storing", dataKey, path));+ write(lockName, () -> {+ try {+ var pathIndexes = PathIndexUtils.read(userService.getWorkspaceDataDir(userId, dataKey, false));+ boolean pathIndexesChanged = false;+ Integer index = pathIndexes.get(path);+ if (index == null) {+ index = pathIndexes.values().stream()+ .mapToInt(Integer::intValue).max().orElse(0) + 1;+ pathIndexes.put(path, index);+ pathIndexesChanged = true;+ }+ var pathStorageFile = new File(dataStorageDir, String.valueOf(index));+ try (var os = new FileOutputStream(pathStorageFile)) {+ TarUtils.tar(pathDir, os, false);+ }+ if (pathIndexesChanged)+ PathIndexUtils.write(userService.getWorkspaceDataDir(userId, dataKey, true), pathIndexes);+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ });+ }+ }+ SiteSyncUtils.bumpVersions(usersStorageDir, dataStorageDir);+ var workspaceServer = getClusterService().getLocalServerAddress();+ getClusterService().submitToAllServers(new SyncTask(workspaceServer, userId, dataKey));+ }+ }++ private static UserService getUserService() {+ return OneDev.getInstance(UserService.class);+ }++ private static ClusterService getClusterService() {+ return OneDev.getInstance(ClusterService.class);+ }++ private static class SyncTask implements ClusterTask<Void> {++ private static final long serialVersionUID = 1L;++ private final String workspaceServer;++ private final Long userId;++ private final String dataKey;++ public SyncTask(String workspaceServer, Long userId, String dataKey) {+ this.workspaceServer = workspaceServer;+ this.userId = userId;+ this.dataKey = dataKey;+ }++ @Override+ public Void call() throws Exception {+ try {+ var localServer = getClusterService().getLocalServerAddress();+ if (!localServer.equals(workspaceServer)) {+ var usersStorageDir = getUsersStorageDir();+ var dataStorageDir = getUserService().getWorkspaceDataDir(userId, dataKey, true);+ var syncPath = Bootstrap.getSiteDir().toPath().relativize(dataStorageDir.toPath()).toString();+ var lockName = User.getWorkspaceDataLockName(userId, dataKey);+ SiteSyncUtils.syncDirectory(workspaceServer, syncPath,+ false, lockName, lockName);+ SiteSyncUtils.bumpVersions(usersStorageDir, dataStorageDir.getParentFile());+ }+ } catch (Exception e) {+ logger.error("Error syncing user data", e);+ }+ return null;+ }++ }+}diff --git a/server-product/pom.xml b/server-product/pom.xmlindex 1ea7618b51..4cb9818a56 100644--- a/server-product/pom.xml+++ b/server-product/pom.xml@@ -130,6 +130,11 @@<artifactId>server-plugin-provisioner-shell</artifactId><version>${project.version}</version></dependency>+ <dependency>+ <groupId>io.onedev</groupId>+ <artifactId>server-plugin-provisioner-docker</artifactId>+ <version>${project.version}</version>+ </dependency><dependency><groupId>io.onedev</groupId><artifactId>server-plugin-authenticator-ldap</artifactId>