The Unified and Autonomous Development Platform
feat: Improve pull repository step to be able to sync non-child projects (#1826)
Robin Shen29aaeadaf749
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java b/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.javaindex aea05344a2..359a6e3f37 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java@@ -25,6 +25,7 @@ import io.onedev.server.buildspec.step.UseTemplateStep;import io.onedev.server.data.migration.VersionedYamlDoc;import io.onedev.server.data.migration.XmlBuildSpecMigrator;import io.onedev.server.job.JobAuthorizationContext;+import io.onedev.server.model.Project;import io.onedev.server.model.support.build.JobProperty;import io.onedev.server.security.SecurityUtils;import io.onedev.server.util.ComponentContext;@@ -1963,5 +1964,34 @@ public class BuildSpec implements Serializable, Validatable {}});}++ private void migrate31(VersionedYamlDoc doc, Stack<Integer> versions) {+ migrateSteps(doc, versions, stepsNode -> {+ for (var itStepNode = stepsNode.getValue().iterator(); itStepNode.hasNext();) {+ MappingNode stepNode = (MappingNode) itStepNode.next();+ if (stepNode.getTag().getValue().equals("!PullRepository")) {+ boolean syncToChild = false;+ String childProject = null;+ for (var itStepTuple = stepNode.getValue().iterator(); itStepTuple.hasNext();) {+ var stepTuple = itStepTuple.next();+ var propName = ((ScalarNode) stepTuple.getKeyNode()).getValue();+ if (propName.equals("syncToChildProject")) {+ syncToChild = Boolean.parseBoolean(((ScalarNode) stepTuple.getValueNode()).getValue());+ itStepTuple.remove();+ } else if (propName.equals("childProject")) {+ childProject = ((ScalarNode) stepTuple.getValueNode()).getValue();+ itStepTuple.remove();+ }+ }+ if (syncToChild) {+ var targetProject = Project.get() + "/" + childProject;+ stepNode.getValue().add(new NodeTuple(+ new ScalarNode(Tag.STR, "targetProject"),+ new ScalarNode(Tag.STR, targetProject)));+ }+ }+ }+ });+ }}diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/PullRepository.java b/server-core/src/main/java/io/onedev/server/buildspec/step/PullRepository.javaindex 9943c66cba..56cabdcc14 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/step/PullRepository.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/PullRepository.java@@ -3,7 +3,6 @@ package io.onedev.server.buildspec.step;import com.google.common.base.Splitter;import com.google.common.collect.MapDifference;import com.google.common.collect.MapDifference.ValueDifference;-import edu.emory.mathcs.backport.java.util.Collections;import io.onedev.commons.bootstrap.Bootstrap;import io.onedev.commons.codeassist.InputSuggestion;import io.onedev.commons.utils.ExplicitException;@@ -17,7 +16,7 @@ import io.onedev.server.OneDev;import io.onedev.server.annotation.ChoiceProvider;import io.onedev.server.annotation.Editable;import io.onedev.server.annotation.Interpolative;-import io.onedev.server.annotation.ShowCondition;+import io.onedev.server.annotation.ProjectChoice;import io.onedev.server.buildspec.BuildSpec;import io.onedev.server.cluster.ClusterTask;import io.onedev.server.entitymanager.BuildManager;@@ -28,10 +27,9 @@ import io.onedev.server.git.CommandUtils;import io.onedev.server.git.GitUtils;import io.onedev.server.git.command.LfsFetchAllCommand;import io.onedev.server.git.command.LfsFetchCommand;-import io.onedev.server.model.Build;import io.onedev.server.model.Project;import io.onedev.server.persistence.SessionManager;-import io.onedev.server.util.EditContext;+import io.onedev.server.security.SecurityUtils;import io.onedev.server.util.match.WildcardUtils;import org.eclipse.jgit.lib.AnyObjectId;import org.eclipse.jgit.lib.ObjectId;@@ -42,7 +40,6 @@ import org.slf4j.LoggerFactory;import javax.annotation.Nullable;import javax.validation.constraints.NotEmpty;import java.io.File;-import java.util.ArrayList;import java.util.HashMap;import java.util.List;import java.util.Map;@@ -62,50 +59,39 @@ public class PullRepository extends SyncRepository {private static final Logger logger = LoggerFactory.getLogger(PullRepository.class);- private boolean syncToChildProject;+ private String targetProject;++ private String accessTokenSecret;- private String childProject;-private String refs = "refs/heads/* refs/tags/*";private boolean withLfs;-- @Editable(order=205, description="If enabled, sync to child project instead of current project")- public boolean isSyncToChildProject() {- return syncToChildProject;- }- public void setSyncToChildProject(boolean syncToChildProject) {- this.syncToChildProject = syncToChildProject;+ @Editable(order=600, placeholder = "Current project", description=+ "Select project to sync to. Leave empty to sync to current project")+ @ProjectChoice+ public String getTargetProject() {+ return targetProject;}- @Editable(order=210, description="Select child project to sync to")- @ShowCondition("isSyncToChildProjectEnabled")- @ChoiceProvider("getChildProjects")- @NotEmpty- public String getChildProject() {- return childProject;+ public void setTargetProject(String targetProject) {+ this.targetProject = targetProject;}- public void setChildProject(String childProject) {- this.childProject = childProject;+ @Editable(order=650, name="Access Token for Target Project", description = "Specify a secret to be used " ++ "as access token with project management permission for target project. Note that access token " ++ "is not required when sync to current or child project and build commit is reachable from " ++ "default branch")+ @ChoiceProvider("getSecretChoices")+ public String getAccessTokenSecret() {+ return accessTokenSecret;}- @SuppressWarnings("unused")- private static boolean isSyncToChildProjectEnabled() {- return (boolean) EditContext.get().getInputValue("syncToChildProject");+ public void setAccessTokenSecret(String accessTokenSecret) {+ this.accessTokenSecret = accessTokenSecret;}- @SuppressWarnings("unused")- private static List<String> getChildProjects() {- int prefixLen = Project.get().getPath().length() + 1;- List<String> choices = new ArrayList<>(Project.get().getDescendants()- .stream().map(pr -> pr.getPath().substring(prefixLen)).collect(toList()));- Collections.sort(choices);- return choices;- }-- @Editable(order=410, description="Specify space separated refs to pull from remote. '*' can be used in ref "+ @Editable(order=700, description="Specify space separated refs to pull from remote. '*' can be used in ref "+ "name for wildcard match<br>"+ "<b class='text-danger'>NOTE:</b> branch/tag protection rule will be ignored when update "+ "branches/tags via this step")@@ -123,7 +109,7 @@ public class PullRepository extends SyncRepository {return BuildSpec.suggestVariables(matchWith, false, false, false);}- @Editable(order=450, name="Transfer LFS Files", descriptionProvider="getLfsDescription")+ @Editable(order=800, name="Transfer LFS Files", descriptionProvider="getLfsDescription")public boolean isWithLfs() {return withLfs;}@@ -146,17 +132,30 @@ public class PullRepository extends SyncRepository {public ServerStepResult run(Long buildId, File inputDir, TaskLogger logger) {return OneDev.getInstance(SessionManager.class).call(() -> {var build = OneDev.getInstance(BuildManager.class).load(buildId);- Project buildProject = build.getProject();- if (!buildProject.isCommitOnBranch(build.getCommitId(), buildProject.getDefaultBranch())) {- logger.error("For security reason, this step is only allowed to run from default branch");- return new ServerStepResult(false);+ Project project = build.getProject();+ Project targetProject;+ if (getTargetProject() != null) {+ targetProject = getProjectManager().findByPath(getTargetProject());+ if (targetProject == null)+ throw new ExplicitException("Target project not found: " + getTargetProject());+ } else {+ targetProject = project;+ }+ boolean authorized = false;+ if (project.isCommitOnBranch(build.getCommitId(), project.getDefaultBranch())+ && project.isSelfOrAncestorOf(targetProject)) {+ authorized = true;+ } else if (getAccessTokenSecret() != null &&+ SecurityUtils.canManageProject(build.getUser(getAccessTokenSecret()), targetProject)) {+ authorized = true;}+ if (!authorized)+ throw new ExplicitException("This build is not authorized to sync to project: " + targetProject.getPath());String remoteUrl = getRemoteUrlWithCredential(build);- Long projectId = getTargetProject(build).getId();-- var task = new PullTask(projectId, remoteUrl, getCertificate(), getRefs(), isForce(), isWithLfs(), getProxy());- getProjectManager().runOnActiveServer(projectId, task);+ Long targetProjectId = targetProject.getId();+ var task = new PullTask(targetProjectId, remoteUrl, getCertificate(), getRefs(), isForce(), isWithLfs(), getProxy());+ getProjectManager().runOnActiveServer(targetProjectId, task);return new ServerStepResult(true);});}@@ -165,19 +164,6 @@ public class PullRepository extends SyncRepository {return OneDev.getInstance(ProjectManager.class);}- private Project getTargetProject(Build build) {- Project buildProject = build.getProject();- if (isSyncToChildProject()) {- String projectPath = buildProject.getPath() + "/" + getChildProject();- Project childProject = OneDev.getInstance(ProjectManager.class).findByPath(projectPath);- if (childProject == null)- throw new ExplicitException("Unable to find child project: " + getChildProject());- return childProject;- } else {- return buildProject;- }- }-private static class PullTask implements ClusterTask<Void> {private final Long projectId;diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/SyncRepository.java b/server-core/src/main/java/io/onedev/server/buildspec/step/SyncRepository.javaindex cf7e53eca5..8380eb72ed 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/step/SyncRepository.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/SyncRepository.java@@ -1,6 +1,5 @@package io.onedev.server.buildspec.step;-import io.onedev.commons.bootstrap.Bootstrap;import io.onedev.commons.codeassist.InputSuggestion;import io.onedev.commons.utils.FileUtils;import io.onedev.commons.utils.StringUtils;@@ -54,7 +53,7 @@ public abstract class SyncRepository extends ServerSideStep {return BuildSpec.suggestVariables(matchWith, false, false, false);}- @Editable(order=300, description="Optionally specify user name to access above repository")+ @Editable(order=300, description="Optionally specify user name to access remote repository")public String getUserName() {return userName;}@@ -63,9 +62,9 @@ public abstract class SyncRepository extends ServerSideStep {this.userName = userName;}- @Editable(order=400, name="Password / Access Token",- description="Specify a secret to be used as password or access token to access above repository")- @ChoiceProvider("getPasswordSecretChoices")+ @Editable(order=400, name="Password or Access Token for Remote Repository",+ description="Specify a secret to be used as password or access token to access remote repository")+ @ChoiceProvider("getSecretChoices")public String getPasswordSecret() {return passwordSecret;}@@ -75,14 +74,14 @@ public abstract class SyncRepository extends ServerSideStep {}@SuppressWarnings("unused")- private static List<String> getPasswordSecretChoices() {+ static List<String> getSecretChoices() {return Project.get().getHierarchyJobSecrets().stream().map(it->it.getName()).collect(Collectors.toList());}@Editable(order=450, name="Certificates to Trust", placeholder = "Base64 encoded PEM format, starting with " +"-----BEGIN CERTIFICATE----- and ending with -----END CERTIFICATE-----",- description = "Specify certificate to trust if you are using self-signed certificate for above url")+ description = "Specify certificate to trust if you are using self-signed certificate for remote repository")@Multiline(monospace = true)@Interpolative(variableSuggester="suggestVariables")public String getCertificate() {@@ -92,18 +91,18 @@ public abstract class SyncRepository extends ServerSideStep {public void setCertificate(String certificate) {this.certificate = certificate;}-- @SuppressWarnings("unused")- private static String getLfsDescription() {- if (!Bootstrap.isInDocker()) {- return "If this option is enabled, git lfs command needs to be installed on OneDev server "- + "(even this step runs on other node)";- } else {- return null;- }++ @Editable(order=500, placeholder = "No proxy", description = "Optionally configure proxy to access remote repository. " ++ "Proxy should be in the format of <proxy host>:<proxy port>")+ public String getProxy() {+ return proxy;}- @Editable(order=500, description="Whether or not use force option to overwrite changes in case ref updating "+ public void setProxy(String proxy) {+ this.proxy = proxy;+ }++ @Editable(order=1000, description="Whether or not use force option to overwrite changes in case ref updating "+ "can not be fast-forwarded")public boolean isForce() {return force;@@ -112,17 +111,7 @@ public abstract class SyncRepository extends ServerSideStep {public void setForce(boolean force) {this.force = force;}-- @Editable(order=600, placeholder = "No proxy", description = "Optionally configure proxy for this step. Proxy " +- "should be in the format of <proxy host>:<proxy port>")- public String getProxy() {- return proxy;- }-- public void setProxy(String proxy) {- this.proxy = proxy;- }-+public String getRemoteUrlWithCredential(Build build) {String encodedPassword = null;if (getPasswordSecret() != null) {diff --git a/server-core/src/main/java/io/onedev/server/model/Build.java b/server-core/src/main/java/io/onedev/server/model/Build.javaindex 9f7551215f..444849d17a 100644--- a/server-core/src/main/java/io/onedev/server/model/Build.java+++ b/server-core/src/main/java/io/onedev/server/model/Build.java@@ -967,7 +967,7 @@ public class Build extends ProjectBelonging|| accessTokenSecret != null && SecurityUtils.canCreateTag(getUser(accessTokenSecret), project, tagName);}- private User getUser(String accessTokenSecret) {+ public User getUser(String accessTokenSecret) {String accessToken = getJobAuthorizationContext().getSecretValue(accessTokenSecret);User user = OneDev.getInstance(UserManager.class).findByAccessToken(accessToken);if (user == null)diff --git a/server-core/src/main/java/io/onedev/server/model/Project.java b/server-core/src/main/java/io/onedev/server/model/Project.javaindex c25d2835d8..d131f9202a 100644--- a/server-core/src/main/java/io/onedev/server/model/Project.java+++ b/server-core/src/main/java/io/onedev/server/model/Project.java@@ -827,14 +827,19 @@ public class Project extends AbstractEntity implements LabelSupport<ProjectLabel@Nullablepublic BuildSpec getBuildSpec(ObjectId commitId) {- if (buildSpecCache == null)- buildSpecCache = new HashMap<>();- Optional<BuildSpec> buildSpec = buildSpecCache.get(commitId);- if (buildSpec == null) {- buildSpec = Optional.fromNullable(loadBuildSpec(commitId));- buildSpecCache.put(commitId, buildSpec);+ Project.push(this);+ try {+ if (buildSpecCache == null)+ buildSpecCache = new HashMap<>();+ Optional<BuildSpec> buildSpec = buildSpecCache.get(commitId);+ if (buildSpec == null) {+ buildSpec = Optional.fromNullable(loadBuildSpec(commitId));+ buildSpecCache.put(commitId, buildSpec);+ }+ return buildSpec.orNull();+ } finally {+ Project.pop();}- return buildSpec.orNull();}public List<String> getJobNames() {diff --git a/server-core/src/main/java/io/onedev/server/security/SecurityUtils.java b/server-core/src/main/java/io/onedev/server/security/SecurityUtils.javaindex 723977966f..f9c90611b8 100644--- a/server-core/src/main/java/io/onedev/server/security/SecurityUtils.java+++ b/server-core/src/main/java/io/onedev/server/security/SecurityUtils.java@@ -199,7 +199,11 @@ public class SecurityUtils extends org.apache.shiro.SecurityUtils {}public static boolean canManageProject(Project project) {- return getSubject().isPermitted(new ProjectPermission(project, new ManageProject()));+ return canManageProject(getUser(), project);+ }++ public static boolean canManageProject(@Nullable User user, Project project) {+ return asSubject(user).isPermitted(new ProjectPermission(project, new ManageProject()));}public static boolean canManageIssues(Project project) {