Skip to content

The Unified and Autonomous Development Platform

feat: Improve pull repository step to be able to sync non-child projects (#1826)

Robin Shen29aaeadaf749
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java b/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java
index aea05344a2..359a6e3f37 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java
@@ -25,6 +25,7 @@ import io.onedev.server.buildspec.step.UseTemplateStep;
import io.onedev.server.data.migration.VersionedYamlDoc;
import io.onedev.server.data.migration.XmlBuildSpecMigrator;
import io.onedev.server.job.JobAuthorizationContext;
+import io.onedev.server.model.Project;
import io.onedev.server.model.support.build.JobProperty;
import io.onedev.server.security.SecurityUtils;
import io.onedev.server.util.ComponentContext;
@@ -1963,5 +1964,34 @@ public class BuildSpec implements Serializable, Validatable {
}
});
}
+
+ private void migrate31(VersionedYamlDoc doc, Stack<Integer> versions) {
+ migrateSteps(doc, versions, stepsNode -> {
+ for (var itStepNode = stepsNode.getValue().iterator(); itStepNode.hasNext();) {
+ MappingNode stepNode = (MappingNode) itStepNode.next();
+ if (stepNode.getTag().getValue().equals("!PullRepository")) {
+ boolean syncToChild = false;
+ String childProject = null;
+ for (var itStepTuple = stepNode.getValue().iterator(); itStepTuple.hasNext();) {
+ var stepTuple = itStepTuple.next();
+ var propName = ((ScalarNode) stepTuple.getKeyNode()).getValue();
+ if (propName.equals("syncToChildProject")) {
+ syncToChild = Boolean.parseBoolean(((ScalarNode) stepTuple.getValueNode()).getValue());
+ itStepTuple.remove();
+ } else if (propName.equals("childProject")) {
+ childProject = ((ScalarNode) stepTuple.getValueNode()).getValue();
+ itStepTuple.remove();
+ }
+ }
+ if (syncToChild) {
+ var targetProject = Project.get() + "/" + childProject;
+ stepNode.getValue().add(new NodeTuple(
+ new ScalarNode(Tag.STR, "targetProject"),
+ new ScalarNode(Tag.STR, targetProject)));
+ }
+ }
+ }
+ });
+ }
}
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/PullRepository.java b/server-core/src/main/java/io/onedev/server/buildspec/step/PullRepository.java
index 9943c66cba..56cabdcc14 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/step/PullRepository.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/PullRepository.java
@@ -3,7 +3,6 @@ package io.onedev.server.buildspec.step;
import com.google.common.base.Splitter;
import com.google.common.collect.MapDifference;
import com.google.common.collect.MapDifference.ValueDifference;
-import edu.emory.mathcs.backport.java.util.Collections;
import io.onedev.commons.bootstrap.Bootstrap;
import io.onedev.commons.codeassist.InputSuggestion;
import io.onedev.commons.utils.ExplicitException;
@@ -17,7 +16,7 @@ import io.onedev.server.OneDev;
import io.onedev.server.annotation.ChoiceProvider;
import io.onedev.server.annotation.Editable;
import io.onedev.server.annotation.Interpolative;
-import io.onedev.server.annotation.ShowCondition;
+import io.onedev.server.annotation.ProjectChoice;
import io.onedev.server.buildspec.BuildSpec;
import io.onedev.server.cluster.ClusterTask;
import io.onedev.server.entitymanager.BuildManager;
@@ -28,10 +27,9 @@ import io.onedev.server.git.CommandUtils;
import io.onedev.server.git.GitUtils;
import io.onedev.server.git.command.LfsFetchAllCommand;
import io.onedev.server.git.command.LfsFetchCommand;
-import io.onedev.server.model.Build;
import io.onedev.server.model.Project;
import io.onedev.server.persistence.SessionManager;
-import io.onedev.server.util.EditContext;
+import io.onedev.server.security.SecurityUtils;
import io.onedev.server.util.match.WildcardUtils;
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.ObjectId;
@@ -42,7 +40,6 @@ import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
import javax.validation.constraints.NotEmpty;
import java.io.File;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -62,50 +59,39 @@ public class PullRepository extends SyncRepository {
private static final Logger logger = LoggerFactory.getLogger(PullRepository.class);
- private boolean syncToChildProject;
+ private String targetProject;
+
+ private String accessTokenSecret;
- private String childProject;
-
private String refs = "refs/heads/* refs/tags/*";
private boolean withLfs;
-
- @Editable(order=205, description="If enabled, sync to child project instead of current project")
- public boolean isSyncToChildProject() {
- return syncToChildProject;
- }
- public void setSyncToChildProject(boolean syncToChildProject) {
- this.syncToChildProject = syncToChildProject;
+ @Editable(order=600, placeholder = "Current project", description=
+ "Select project to sync to. Leave empty to sync to current project")
+ @ProjectChoice
+ public String getTargetProject() {
+ return targetProject;
}
- @Editable(order=210, description="Select child project to sync to")
- @ShowCondition("isSyncToChildProjectEnabled")
- @ChoiceProvider("getChildProjects")
- @NotEmpty
- public String getChildProject() {
- return childProject;
+ public void setTargetProject(String targetProject) {
+ this.targetProject = targetProject;
}
- public void setChildProject(String childProject) {
- this.childProject = childProject;
+ @Editable(order=650, name="Access Token for Target Project", description = "Specify a secret to be used " +
+ "as access token with project management permission for target project. Note that access token " +
+ "is not required when sync to current or child project and build commit is reachable from " +
+ "default branch")
+ @ChoiceProvider("getSecretChoices")
+ public String getAccessTokenSecret() {
+ return accessTokenSecret;
}
- @SuppressWarnings("unused")
- private static boolean isSyncToChildProjectEnabled() {
- return (boolean) EditContext.get().getInputValue("syncToChildProject");
+ public void setAccessTokenSecret(String accessTokenSecret) {
+ this.accessTokenSecret = accessTokenSecret;
}
- @SuppressWarnings("unused")
- private static List<String> getChildProjects() {
- int prefixLen = Project.get().getPath().length() + 1;
- List<String> choices = new ArrayList<>(Project.get().getDescendants()
- .stream().map(pr -> pr.getPath().substring(prefixLen)).collect(toList()));
- Collections.sort(choices);
- return choices;
- }
-
- @Editable(order=410, description="Specify space separated refs to pull from remote. '*' can be used in ref "
+ @Editable(order=700, description="Specify space separated refs to pull from remote. '*' can be used in ref "
+ "name for wildcard match<br>"
+ "<b class='text-danger'>NOTE:</b> branch/tag protection rule will be ignored when update "
+ "branches/tags via this step")
@@ -123,7 +109,7 @@ public class PullRepository extends SyncRepository {
return BuildSpec.suggestVariables(matchWith, false, false, false);
}
- @Editable(order=450, name="Transfer LFS Files", descriptionProvider="getLfsDescription")
+ @Editable(order=800, name="Transfer LFS Files", descriptionProvider="getLfsDescription")
public boolean isWithLfs() {
return withLfs;
}
@@ -146,17 +132,30 @@ public class PullRepository extends SyncRepository {
public ServerStepResult run(Long buildId, File inputDir, TaskLogger logger) {
return OneDev.getInstance(SessionManager.class).call(() -> {
var build = OneDev.getInstance(BuildManager.class).load(buildId);
- Project buildProject = build.getProject();
- if (!buildProject.isCommitOnBranch(build.getCommitId(), buildProject.getDefaultBranch())) {
- logger.error("For security reason, this step is only allowed to run from default branch");
- return new ServerStepResult(false);
+ Project project = build.getProject();
+ Project targetProject;
+ if (getTargetProject() != null) {
+ targetProject = getProjectManager().findByPath(getTargetProject());
+ if (targetProject == null)
+ throw new ExplicitException("Target project not found: " + getTargetProject());
+ } else {
+ targetProject = project;
+ }
+ boolean authorized = false;
+ if (project.isCommitOnBranch(build.getCommitId(), project.getDefaultBranch())
+ && project.isSelfOrAncestorOf(targetProject)) {
+ authorized = true;
+ } else if (getAccessTokenSecret() != null &&
+ SecurityUtils.canManageProject(build.getUser(getAccessTokenSecret()), targetProject)) {
+ authorized = true;
}
+ if (!authorized)
+ throw new ExplicitException("This build is not authorized to sync to project: " + targetProject.getPath());
String remoteUrl = getRemoteUrlWithCredential(build);
- Long projectId = getTargetProject(build).getId();
-
- var task = new PullTask(projectId, remoteUrl, getCertificate(), getRefs(), isForce(), isWithLfs(), getProxy());
- getProjectManager().runOnActiveServer(projectId, task);
+ Long targetProjectId = targetProject.getId();
+ var task = new PullTask(targetProjectId, remoteUrl, getCertificate(), getRefs(), isForce(), isWithLfs(), getProxy());
+ getProjectManager().runOnActiveServer(targetProjectId, task);
return new ServerStepResult(true);
});
}
@@ -165,19 +164,6 @@ public class PullRepository extends SyncRepository {
return OneDev.getInstance(ProjectManager.class);
}
- private Project getTargetProject(Build build) {
- Project buildProject = build.getProject();
- if (isSyncToChildProject()) {
- String projectPath = buildProject.getPath() + "/" + getChildProject();
- Project childProject = OneDev.getInstance(ProjectManager.class).findByPath(projectPath);
- if (childProject == null)
- throw new ExplicitException("Unable to find child project: " + getChildProject());
- return childProject;
- } else {
- return buildProject;
- }
- }
-
private static class PullTask implements ClusterTask<Void> {
private final Long projectId;
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/SyncRepository.java b/server-core/src/main/java/io/onedev/server/buildspec/step/SyncRepository.java
index cf7e53eca5..8380eb72ed 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/step/SyncRepository.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/SyncRepository.java
@@ -1,6 +1,5 @@
package io.onedev.server.buildspec.step;
-import io.onedev.commons.bootstrap.Bootstrap;
import io.onedev.commons.codeassist.InputSuggestion;
import io.onedev.commons.utils.FileUtils;
import io.onedev.commons.utils.StringUtils;
@@ -54,7 +53,7 @@ public abstract class SyncRepository extends ServerSideStep {
return BuildSpec.suggestVariables(matchWith, false, false, false);
}
- @Editable(order=300, description="Optionally specify user name to access above repository")
+ @Editable(order=300, description="Optionally specify user name to access remote repository")
public String getUserName() {
return userName;
}
@@ -63,9 +62,9 @@ public abstract class SyncRepository extends ServerSideStep {
this.userName = userName;
}
- @Editable(order=400, name="Password / Access Token",
- description="Specify a secret to be used as password or access token to access above repository")
- @ChoiceProvider("getPasswordSecretChoices")
+ @Editable(order=400, name="Password or Access Token for Remote Repository",
+ description="Specify a secret to be used as password or access token to access remote repository")
+ @ChoiceProvider("getSecretChoices")
public String getPasswordSecret() {
return passwordSecret;
}
@@ -75,14 +74,14 @@ public abstract class SyncRepository extends ServerSideStep {
}
@SuppressWarnings("unused")
- private static List<String> getPasswordSecretChoices() {
+ static List<String> getSecretChoices() {
return Project.get().getHierarchyJobSecrets()
.stream().map(it->it.getName()).collect(Collectors.toList());
}
@Editable(order=450, name="Certificates to Trust", placeholder = "Base64 encoded PEM format, starting with " +
"-----BEGIN CERTIFICATE----- and ending with -----END CERTIFICATE-----",
- description = "Specify certificate to trust if you are using self-signed certificate for above url")
+ description = "Specify certificate to trust if you are using self-signed certificate for remote repository")
@Multiline(monospace = true)
@Interpolative(variableSuggester="suggestVariables")
public String getCertificate() {
@@ -92,18 +91,18 @@ public abstract class SyncRepository extends ServerSideStep {
public void setCertificate(String certificate) {
this.certificate = certificate;
}
-
- @SuppressWarnings("unused")
- private static String getLfsDescription() {
- if (!Bootstrap.isInDocker()) {
- return "If this option is enabled, git lfs command needs to be installed on OneDev server "
- + "(even this step runs on other node)";
- } else {
- return null;
- }
+
+ @Editable(order=500, placeholder = "No proxy", description = "Optionally configure proxy to access remote repository. " +
+ "Proxy should be in the format of &lt;proxy host&gt;:&lt;proxy port&gt;")
+ public String getProxy() {
+ return proxy;
}
- @Editable(order=500, description="Whether or not use force option to overwrite changes in case ref updating "
+ public void setProxy(String proxy) {
+ this.proxy = proxy;
+ }
+
+ @Editable(order=1000, description="Whether or not use force option to overwrite changes in case ref updating "
+ "can not be fast-forwarded")
public boolean isForce() {
return force;
@@ -112,17 +111,7 @@ public abstract class SyncRepository extends ServerSideStep {
public void setForce(boolean force) {
this.force = force;
}
-
- @Editable(order=600, placeholder = "No proxy", description = "Optionally configure proxy for this step. Proxy " +
- "should be in the format of &lt;proxy host&gt;:&lt;proxy port&gt;")
- public String getProxy() {
- return proxy;
- }
-
- public void setProxy(String proxy) {
- this.proxy = proxy;
- }
-
+
public String getRemoteUrlWithCredential(Build build) {
String encodedPassword = null;
if (getPasswordSecret() != null) {
diff --git a/server-core/src/main/java/io/onedev/server/model/Build.java b/server-core/src/main/java/io/onedev/server/model/Build.java
index 9f7551215f..444849d17a 100644
--- a/server-core/src/main/java/io/onedev/server/model/Build.java
+++ b/server-core/src/main/java/io/onedev/server/model/Build.java
@@ -967,7 +967,7 @@ public class Build extends ProjectBelonging
|| accessTokenSecret != null && SecurityUtils.canCreateTag(getUser(accessTokenSecret), project, tagName);
}
- private User getUser(String accessTokenSecret) {
+ public User getUser(String accessTokenSecret) {
String accessToken = getJobAuthorizationContext().getSecretValue(accessTokenSecret);
User user = OneDev.getInstance(UserManager.class).findByAccessToken(accessToken);
if (user == null)
diff --git a/server-core/src/main/java/io/onedev/server/model/Project.java b/server-core/src/main/java/io/onedev/server/model/Project.java
index c25d2835d8..d131f9202a 100644
--- a/server-core/src/main/java/io/onedev/server/model/Project.java
+++ b/server-core/src/main/java/io/onedev/server/model/Project.java
@@ -827,14 +827,19 @@ public class Project extends AbstractEntity implements LabelSupport<ProjectLabel
@Nullable
public BuildSpec getBuildSpec(ObjectId commitId) {
- if (buildSpecCache == null)
- buildSpecCache = new HashMap<>();
- Optional<BuildSpec> buildSpec = buildSpecCache.get(commitId);
- if (buildSpec == null) {
- buildSpec = Optional.fromNullable(loadBuildSpec(commitId));
- buildSpecCache.put(commitId, buildSpec);
+ Project.push(this);
+ try {
+ if (buildSpecCache == null)
+ buildSpecCache = new HashMap<>();
+ Optional<BuildSpec> buildSpec = buildSpecCache.get(commitId);
+ if (buildSpec == null) {
+ buildSpec = Optional.fromNullable(loadBuildSpec(commitId));
+ buildSpecCache.put(commitId, buildSpec);
+ }
+ return buildSpec.orNull();
+ } finally {
+ Project.pop();
}
- return buildSpec.orNull();
}
public List<String> getJobNames() {
diff --git a/server-core/src/main/java/io/onedev/server/security/SecurityUtils.java b/server-core/src/main/java/io/onedev/server/security/SecurityUtils.java
index 723977966f..f9c90611b8 100644
--- a/server-core/src/main/java/io/onedev/server/security/SecurityUtils.java
+++ b/server-core/src/main/java/io/onedev/server/security/SecurityUtils.java
@@ -199,7 +199,11 @@ public class SecurityUtils extends org.apache.shiro.SecurityUtils {
}
public static boolean canManageProject(Project project) {
- return getSubject().isPermitted(new ProjectPermission(project, new ManageProject()));
+ return canManageProject(getUser(), project);
+ }
+
+ public static boolean canManageProject(@Nullable User user, Project project) {
+ return asSubject(user).isPermitted(new ProjectPermission(project, new ManageProject()));
}
public static boolean canManageIssues(Project project) {