The Unified and Autonomous Development Platform
Specify registry login instead of image pull secret in Kubernetes executor for easy use
Robin Shen3ec2c255edb6
diff --git a/server-core/src/main/java/io/onedev/server/cache/DefaultBuildInfoManager.java b/server-core/src/main/java/io/onedev/server/cache/DefaultBuildInfoManager.javaindex f4b3c9deee..52bca3a5c0 100644--- a/server-core/src/main/java/io/onedev/server/cache/DefaultBuildInfoManager.java+++ b/server-core/src/main/java/io/onedev/server/cache/DefaultBuildInfoManager.java@@ -115,7 +115,7 @@ public class DefaultBuildInfoManager extends AbstractEnvironmentManager implemen}private boolean collect(Project project) {- logger.debug("Collecting build info in project '{}'...", project);+ logger.debug("Collecting build info (project: {})...", project);Environment env = getEnv(project.getId().toString());Store defaultStore = getStore(env, DEFAULT_STORE);@@ -164,7 +164,7 @@ public class DefaultBuildInfoManager extends AbstractEnvironmentManager implemen});}-+ logger.debug("Collected build info (project: {})", project);return unprocessedBuilds.size() == BATCH_SIZE;}diff --git a/server-core/src/main/java/io/onedev/server/cache/DefaultCodeCommentRelationInfoManager.java b/server-core/src/main/java/io/onedev/server/cache/DefaultCodeCommentRelationInfoManager.javaindex d7c33d38fc..772214928e 100644--- a/server-core/src/main/java/io/onedev/server/cache/DefaultCodeCommentRelationInfoManager.java+++ b/server-core/src/main/java/io/onedev/server/cache/DefaultCodeCommentRelationInfoManager.java@@ -140,7 +140,7 @@ public class DefaultCodeCommentRelationInfoManager extends AbstractEnvironmentMa}private boolean collect(Project project) {- logger.debug("Collecting code comment relation info in project '{}'...", project);+ logger.debug("Collecting code comment relation info (project: {})...", project);Environment env = getEnv(project.getId().toString());Store defaultStore = getStore(env, DEFAULT_STORE);@@ -264,6 +264,7 @@ public class DefaultCodeCommentRelationInfoManager extends AbstractEnvironmentMa});}}+ logger.debug("Collected code comment relation info (project: {})", project);return unprocessedPullRequestUpdates.size() == BATCH_SIZE || unprocessedCodeComments.size() == BATCH_SIZE;}diff --git a/server-core/src/main/java/io/onedev/server/cache/DefaultCommitInfoManager.java b/server-core/src/main/java/io/onedev/server/cache/DefaultCommitInfoManager.javaindex 04eb61e3c3..2752894336 100644--- a/server-core/src/main/java/io/onedev/server/cache/DefaultCommitInfoManager.java+++ b/server-core/src/main/java/io/onedev/server/cache/DefaultCommitInfoManager.java@@ -314,6 +314,8 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme}private void doCollect(Project project, ObjectId commitId, String refName) {+ logger.debug("Collecting commit information (project: {}, ref: {})...", refName, project.getName());+Environment env = getEnv(project.getId().toString());Store defaultStore = getStore(env, DEFAULT_STORE);Store commitsStore = getStore(env, COMMITS_STORE);@@ -762,6 +764,7 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager impleme}}}+ logger.debug("Collected commit information (project: {}, ref: {})", project.getName(), refName);}private void updateContribution(Map<Integer, Contribution> contributions, int key, GitCommit commit) {@@ -991,11 +994,8 @@ public class DefaultCommitInfoManager extends AbstractEnvironmentManager implemecollectingWorks.add((CollectingWork)work);Collections.sort(collectingWorks, new CommitTimeComparator());- for (CollectingWork work: collectingWorks) {- logger.debug("Collecting commit information up to ref '{}' in project '{}'...",- work.getRefName(), project.getName());+ for (CollectingWork work: collectingWorks)doCollect(project, work.getCommit().copy(), work.getRefName());- }}});diff --git a/server-core/src/main/java/io/onedev/server/model/support/RegistryLogin.java b/server-core/src/main/java/io/onedev/server/model/support/RegistryLogin.javanew file mode 100644index 0000000000..644d810490--- /dev/null+++ b/server-core/src/main/java/io/onedev/server/model/support/RegistryLogin.java@@ -0,0 +1,53 @@+package io.onedev.server.model.support;++import java.io.Serializable;++import org.hibernate.validator.constraints.NotEmpty;++import io.onedev.server.web.editable.annotation.Editable;+import io.onedev.server.web.editable.annotation.NameOfEmptyValue;+import io.onedev.server.web.editable.annotation.Password;++@Editable+public class RegistryLogin implements Serializable {++ private static final long serialVersionUID = 1L;++ private String registryUrl;++ private String userName;++ private String password;++ @Editable(order=100, description="Specify registry url. Leave empty for official registry")+ @NameOfEmptyValue("Default Registry")+ public String getRegistryUrl() {+ return registryUrl;+ }++ public void setRegistryUrl(String registryUrl) {+ this.registryUrl = registryUrl;+ }++ @Editable(order=200)+ @NotEmpty+ public String getUserName() {+ return userName;+ }++ public void setUserName(String userName) {+ this.userName = userName;+ }++ @Editable(order=300)+ @NotEmpty+ @Password+ public String getPassword() {+ return password;+ }++ public void setPassword(String password) {+ this.password = password;+ }++}\ No newline at end of filediff --git a/server-plugin/server-plugin-kubernetes/src/main/java/io/onedev/server/plugin/kubernetes/KubernetesExecutor.java b/server-plugin/server-plugin-kubernetes/src/main/java/io/onedev/server/plugin/kubernetes/KubernetesExecutor.javaindex 060f281372..4b9b0498e4 100644--- a/server-plugin/server-plugin-kubernetes/src/main/java/io/onedev/server/plugin/kubernetes/KubernetesExecutor.java+++ b/server-plugin/server-plugin-kubernetes/src/main/java/io/onedev/server/plugin/kubernetes/KubernetesExecutor.java@@ -5,6 +5,7 @@ import java.io.IOException;import java.io.Serializable;import java.util.ArrayList;import java.util.Base64;+import java.util.Base64.Encoder;import java.util.Collection;import java.util.Iterator;import java.util.LinkedHashMap;@@ -21,10 +22,10 @@ import org.slf4j.Logger;import org.slf4j.LoggerFactory;import org.yaml.snakeyaml.Yaml;+import com.fasterxml.jackson.core.JsonProcessingException;import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import com.google.common.base.Preconditions;-import com.google.common.base.Splitter;import com.google.common.collect.Lists;import com.google.common.collect.Sets;@@ -42,6 +43,7 @@ import io.onedev.server.ci.job.CacheSpec;import io.onedev.server.ci.job.JobContext;import io.onedev.server.entitymanager.SettingManager;import io.onedev.server.model.support.JobExecutor;+import io.onedev.server.model.support.RegistryLogin;import io.onedev.server.plugin.kubernetes.KubernetesExecutor.TestData;import io.onedev.server.util.JobLogger;import io.onedev.server.util.inputspec.SecretInput;@@ -71,7 +73,7 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestDataprivate List<NodeSelectorEntry> nodeSelector = new ArrayList<>();- private String imagePullSecrets;+ private List<RegistryLogin> registryLogins = new ArrayList<>();private String serviceAccount;@@ -125,16 +127,14 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestDatathis.nodeSelector = nodeSelector;}- @Editable(order=22000, group="More Settings", description="Optionally specify space-separated image "- + "pull secrets in above namespace for job pods to access private docker registries. "- + "Refer to <a href='https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/'>kubernetes "- + "documentation</a> on how to set up image pull secrets")- public String getImagePullSecrets() {- return imagePullSecrets;+ @Editable(order=22000, group="More Settings", description="Specify login information of docker registries if necessary. These "+ + "logins will be used to create image pull secrets of the job pods")+ public List<RegistryLogin> getRegistryLogins() {+ return registryLogins;}- public void setImagePullSecrets(String imagePullSecrets) {- this.imagePullSecrets = imagePullSecrets;+ public void setRegistryLogins(List<RegistryLogin> registryLogins) {+ this.registryLogins = registryLogins;}@Editable(order=23000, group="More Settings", description="Optionally specify a service account in above namespace to run the job "@@ -200,6 +200,7 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestDatafile = File.createTempFile("k8s", ".yaml");String resourceYaml = new Yaml().dump(resourceDef);+String maskedYaml = resourceYaml;for (String secret: secretsToMask)maskedYaml = StringUtils.replace(maskedYaml, secret, SecretInput.MASK);@@ -295,15 +296,6 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData}}- private List<Object> getImagePullSecretsData() {- List<Object> data = new ArrayList<>();- if (getImagePullSecrets() != null) {- for (String imagePullSecret: Splitter.on(" ").trimResults().omitEmptyStrings().split(getImagePullSecrets()))- data.add(Maps.newLinkedHashMap("name", imagePullSecret));- }- return data;- }-@Nullableprivate Map<Object, Object> getAffinity(@Nullable JobContext jobContext) {Map<Object, Object> nodeAffinity = new LinkedHashMap<>();@@ -408,13 +400,41 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestDataelsereturn "C:\\ProgramData\\onedev-ci\\cache";}++ @Nullable+ private String createImagePullSecret(JobLogger logger) {+ if (!getRegistryLogins().isEmpty()) {+ Encoder encoder = Base64.getEncoder();+ Map<Object, Object> auths = new LinkedHashMap<>();+ for (RegistryLogin login: getRegistryLogins()) {+ String auth = login.getUserName() + ":" + login.getPassword();+ String registryUrl = login.getRegistryUrl();+ if (registryUrl == null)+ registryUrl = "https://index.docker.io/v1/";+ auths.put(registryUrl, Maps.newLinkedHashMap(+ "auth", encoder.encodeToString(auth.getBytes(Charsets.UTF_8))));+ }+ try {+ String dockerConfig = new ObjectMapper().writeValueAsString(Maps.newLinkedHashMap("auths", auths));+ return createSecret(Maps.newLinkedHashMap(".dockerconfigjson", dockerConfig), "kubernetes.io/dockerconfigjson", logger);+ } catch (JsonProcessingException e) {+ throw new RuntimeException(e);+ }+ } else {+ return null;+ }+ }private void execute(String dockerImage, String jobToken, JobLogger logger, @Nullable JobContext jobContext) {createNamespaceIfNotExist(logger);- Map<String, String> secrets = Maps.newLinkedHashMap(KubernetesHelper.ENV_JOB_TOKEN, jobToken);- String secretName = createSecret(secrets, logger);+ String jobSecretName = null;+ String imagePullSecretName = null;try {+ Map<String, String> jobSecrets = Maps.newLinkedHashMap(KubernetesHelper.ENV_JOB_TOKEN, jobToken);+ jobSecretName = createSecret(jobSecrets, null, logger);+ imagePullSecretName = createImagePullSecret(logger);+String osName = getOSName(logger);Map<String, Object> podSpec = new LinkedHashMap<>();@@ -460,7 +480,7 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData"name", KubernetesHelper.ENV_SERVER_URL,"value", getServerUrl());envs.add(serverUrlEnv);- envs.addAll(getSecretEnvs(secretName, secrets.keySet()));+ envs.addAll(getSecretEnvs(jobSecretName, jobSecrets.keySet()));List<String> sidecarArgs = Lists.newArrayList("-classpath", k8sHelperClassPath, "io.onedev.k8shelper.SideCar");List<String> initArgs = Lists.newArrayList("-classpath", k8sHelperClassPath, "io.onedev.k8shelper.Init");@@ -491,9 +511,8 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestDataif (affinity != null)podSpec.put("affinity", affinity);- List<Object> imagePullSecretsData = getImagePullSecretsData();- if (!imagePullSecretsData.isEmpty())- podSpec.put("imagePullSecrets", imagePullSecretsData);+ if (imagePullSecretName != null)+ podSpec.put("imagePullSecrets", Lists.<Object>newArrayList(Maps.newLinkedHashMap("name", imagePullSecretName)));if (getServiceAccount() != null)podSpec.put("serviceAccountName", getServiceAccount());podSpec.put("restartPolicy", "Never");@@ -642,7 +661,10 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestDatadeleteResource("pod", podName, logger);}} finally {- deleteResource("secret", secretName, logger);+ if (jobSecretName != null)+ deleteResource("secret", jobSecretName, logger);+ if (imagePullSecretName != null)+ deleteResource("secret", imagePullSecretName, logger);}}@@ -794,7 +816,7 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData}}- private String createSecret(Map<String, String> secrets, JobLogger logger) {+ private String createSecret(Map<String, String> secrets, @Nullable String type, JobLogger logger) {Map<String, String> encodedSecrets = new LinkedHashMap<>();for (Map.Entry<String, String> entry: secrets.entrySet())encodedSecrets.put(entry.getKey(), Base64.getEncoder().encodeToString(entry.getValue().getBytes(Charsets.UTF_8)));@@ -805,6 +827,8 @@ public class KubernetesExecutor extends JobExecutor implements Testable<TestData"generateName", "secret-","namespace", getNamespace()),"data", encodedSecrets);+ if (type != null)+ secretDef.put("type", type);return createResource(secretDef, encodedSecrets.values(), logger);}diff --git a/server-plugin/server-plugin-serverdocker/src/main/java/io/onedev/server/plugin/serverdocker/ServerDockerExecutor.java b/server-plugin/server-plugin-serverdocker/src/main/java/io/onedev/server/plugin/serverdocker/ServerDockerExecutor.javaindex 4b2fe1adb5..3d039478e5 100644--- a/server-plugin/server-plugin-serverdocker/src/main/java/io/onedev/server/plugin/serverdocker/ServerDockerExecutor.java+++ b/server-plugin/server-plugin-serverdocker/src/main/java/io/onedev/server/plugin/serverdocker/ServerDockerExecutor.java@@ -40,6 +40,7 @@ import io.onedev.server.OneDev;import io.onedev.server.ci.job.JobContext;import io.onedev.server.ci.job.JobManager;import io.onedev.server.model.support.JobExecutor;+import io.onedev.server.model.support.RegistryLogin;import io.onedev.server.plugin.serverdocker.ServerDockerExecutor.TestData;import io.onedev.server.util.JobLogger;import io.onedev.server.util.OneContext;@@ -48,7 +49,6 @@ import io.onedev.server.util.validation.annotation.ClassValidating;import io.onedev.server.web.editable.annotation.Editable;import io.onedev.server.web.editable.annotation.NameOfEmptyValue;import io.onedev.server.web.editable.annotation.OmitName;-import io.onedev.server.web.editable.annotation.Password;import io.onedev.server.web.util.Testable;@Editable(order=100, description="This executor interpretates job environments as docker images, "@@ -496,50 +496,6 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa}}- @Editable- public static class RegistryLogin implements Serializable {-- private static final long serialVersionUID = 1L;-- private String registryUrl;-- private String userName;-- private String password;-- @Editable(order=100, description="Specify registry url. Leave empty for official registry")- @NameOfEmptyValue("Default Registry")- public String getRegistryUrl() {- return registryUrl;- }-- public void setRegistryUrl(String registryUrl) {- this.registryUrl = registryUrl;- }-- @Editable(order=200)- @NotEmpty- public String getUserName() {- return userName;- }-- public void setUserName(String userName) {- this.userName = userName;- }-- @Editable(order=300)- @NotEmpty- @Password- public String getPassword() {- return password;- }-- public void setPassword(String password) {- this.password = password;- }-- }-@Editable(name="Specify a Docker Image to Test Against")public static class TestData implements Serializable {