Skip to content

The Unified and Autonomous Development Platform

fix: pull from git mirror fails because self-signed cert (#1405)

Robin Shen089ce8e8856f
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/PullRepository.java b/server-core/src/main/java/io/onedev/server/buildspec/step/PullRepository.java
index ccce9a2bcd..05d8cdd28c 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/step/PullRepository.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/PullRepository.java
@@ -7,6 +7,7 @@ import edu.emory.mathcs.backport.java.util.Collections;
import io.onedev.commons.bootstrap.Bootstrap;
import io.onedev.commons.codeassist.InputSuggestion;
import io.onedev.commons.utils.ExplicitException;
+import io.onedev.commons.utils.FileUtils;
import io.onedev.commons.utils.StringUtils;
import io.onedev.commons.utils.TaskLogger;
import io.onedev.commons.utils.command.Commandline;
@@ -149,7 +150,7 @@ public class PullRepository extends SyncRepository {
String remoteUrl = getRemoteUrlWithCredential(build);
Long projectId = getTargetProject(build).getId();
- var task = new PullTask(projectId, remoteUrl, getRefs(), isForce(), isWithLfs(), getProxy());
+ var task = new PullTask(projectId, remoteUrl, getCertificate(), getRefs(), isForce(), isWithLfs(), getProxy());
getProjectManager().runOnActiveServer(projectId, task);
return null;
@@ -178,6 +179,8 @@ public class PullRepository extends SyncRepository {
private final String remoteUrl;
+ private final String certificate;
+
private final String refs;
private final boolean force;
@@ -186,10 +189,11 @@ public class PullRepository extends SyncRepository {
private final String proxy;
- PullTask(Long projectId, String remoteUrl, String refs, boolean force,
- boolean withLfs, @Nullable String proxy) {
+ PullTask(Long projectId, String remoteUrl, @Nullable String certificate,
+ String refs, boolean force, boolean withLfs, @Nullable String proxy) {
this.projectId = projectId;
this.remoteUrl = remoteUrl;
+ this.certificate = certificate;
this.refs = refs;
this.force = force;
this.withLfs = withLfs;
@@ -214,88 +218,63 @@ public class PullRepository extends SyncRepository {
@Override
public Void call() throws Exception {
- Repository repository = getProjectManager().getRepository(projectId);
-
- String defaultBranch = GitUtils.getDefaultBranch(repository);
- Map<String, ObjectId> oldCommitIds = getRefCommits(repository);
-
- Commandline git = CommandUtils.newGit();
- configureProxy(git, proxy);
- git.workingDir(repository.getDirectory());
-
- git.addArgs("fetch");
- git.addArgs(remoteUrl);
- if (force)
- git.addArgs("--force");
-
- for (String each: Splitter.on(' ').omitEmptyStrings().trimResults().split(refs))
- git.addArgs(each + ":" + each);
-
- git.execute(new LineConsumer() {
-
- @Override
- public void consume(String line) {
- logger.debug(line);
- }
-
- }, new LineConsumer() {
-
- @Override
- public void consume(String line) {
- if (!line.startsWith("From") && !line.contains("->"))
- logger.error(line);
- else
- logger.debug(line);
- }
+ var certificateFile = writeCertificate(certificate);
+ try {
+ Repository repository = getProjectManager().getRepository(projectId);
- }).checkReturnCode();
+ String defaultBranch = GitUtils.getDefaultBranch(repository);
+ Map<String, ObjectId> oldCommitIds = getRefCommits(repository);
- Map<String, ObjectId> newCommitIds = getRefCommits(repository);
+ Commandline git = CommandUtils.newGit();
+ configureProxy(git, proxy);
+ configureCertificate(git, certificateFile);
+ git.workingDir(repository.getDirectory());
- if (defaultBranch == null) {
- logger.debug("Determining remote head branch...");
+ git.addArgs("fetch");
+ git.addArgs(remoteUrl);
+ if (force)
+ git.addArgs("--force");
- git.clearArgs();
- configureProxy(git, proxy);
- git.addArgs("remote", "show", remoteUrl);
+ for (String each : Splitter.on(' ').omitEmptyStrings().trimResults().split(refs))
+ git.addArgs(each + ":" + each);
- AtomicReference<String> headBranch = new AtomicReference<>(null);
git.execute(new LineConsumer() {
@Override
public void consume(String line) {
logger.debug(line);
- if (line.trim().startsWith("HEAD branch:"))
- headBranch.set(line.trim().substring("HEAD branch:".length()).trim());
}
}, new LineConsumer() {
@Override
public void consume(String line) {
- logger.warn(line);
+ if (!line.startsWith("From") && !line.contains("->"))
+ logger.error(line);
+ else
+ logger.debug(line);
}
}).checkReturnCode();
- if (headBranch.get() != null) {
- if (GitUtils.resolve(repository, Constants.R_HEADS + headBranch.get(), false) == null) {
- logger.debug("Remote head branch not synced, using first branch as default");
- headBranch.set(null);
- }
- } else {
- logger.debug("Remote head branch not found, using first branch as default");
- }
- if (headBranch.get() == null) {
+ Map<String, ObjectId> newCommitIds = getRefCommits(repository);
+
+ if (defaultBranch == null) {
+ logger.debug("Determining remote head branch...");
+
git.clearArgs();
- git.addArgs("branch");
+ configureProxy(git, proxy);
+ configureCertificate(git, certificateFile);
+ git.addArgs("remote", "show", remoteUrl);
+ AtomicReference<String> headBranch = new AtomicReference<>(null);
git.execute(new LineConsumer() {
@Override
public void consume(String line) {
- if (headBranch.get() == null)
- headBranch.set(StringUtils.stripStart(line.trim(), "*").trim());
+ logger.debug(line);
+ if (line.trim().startsWith("HEAD branch:"))
+ headBranch.set(line.trim().substring("HEAD branch:".length()).trim());
}
}, new LineConsumer() {
@@ -306,62 +285,96 @@ public class PullRepository extends SyncRepository {
}
}).checkReturnCode();
- }
- if (headBranch.get() != null)
- GitUtils.setDefaultBranch(repository, headBranch.get());
- }
-
- if (withLfs) {
- git.clearArgs();
- configureProxy(git, proxy);
- var sinceCommitIds = getProjectManager().readLfsSinceCommits(projectId);
- if (sinceCommitIds.isEmpty()) {
- new LfsFetchAllCommand(git.workingDir(), remoteUrl) {
- protected Commandline newGit() {
- return git;
- }
- }.run();
- } else {
- var fetchCommitIds = getReachableCommits(repository, sinceCommitIds, newCommitIds.values())
- .stream().map(AnyObjectId::copy).collect(toList());
- new LfsFetchCommand(git.workingDir(), remoteUrl, fetchCommitIds) {
- @Override
- protected Commandline newGit() {
- return git;
+ if (headBranch.get() != null) {
+ if (GitUtils.resolve(repository, Constants.R_HEADS + headBranch.get(), false) == null) {
+ logger.debug("Remote head branch not synced, using first branch as default");
+ headBranch.set(null);
}
- }.run();
- }
- getProjectManager().writeLfsSinceCommits(projectId, newCommitIds.values());
- }
+ } else {
+ logger.debug("Remote head branch not found, using first branch as default");
+ }
+ if (headBranch.get() == null) {
+ git.clearArgs();
+ git.addArgs("branch");
+
+ git.execute(new LineConsumer() {
+
+ @Override
+ public void consume(String line) {
+ if (headBranch.get() == null)
+ headBranch.set(StringUtils.stripStart(line.trim(), "*").trim());
+ }
+
+ }, new LineConsumer() {
+
+ @Override
+ public void consume(String line) {
+ logger.warn(line);
+ }
- OneDev.getInstance(SessionManager.class).runAsync(() -> {
- try {
- // Access db connection in a separate thread to avoid possible deadlock, as
- // the parent thread is blocking another thread holding database connections
- var project = getProjectManager().load(projectId);
- MapDifference<String, ObjectId> difference = difference(oldCommitIds, newCommitIds);
- ListenerRegistry registry = OneDev.getInstance(ListenerRegistry.class);
- for (Map.Entry<String, ObjectId> entry : difference.entriesOnlyOnLeft().entrySet()) {
- if (RefUpdated.isValidRef(entry.getKey()))
- registry.post(new RefUpdated(project, entry.getKey(), entry.getValue(), ObjectId.zeroId()));
+ }).checkReturnCode();
}
- for (Map.Entry<String, ObjectId> entry : difference.entriesOnlyOnRight().entrySet()) {
- if (RefUpdated.isValidRef(entry.getKey()))
- registry.post(new RefUpdated(project, entry.getKey(), ObjectId.zeroId(), entry.getValue()));
+ if (headBranch.get() != null)
+ GitUtils.setDefaultBranch(repository, headBranch.get());
+ }
+
+ if (withLfs) {
+ git.clearArgs();
+ configureProxy(git, proxy);
+ configureCertificate(git, certificateFile);
+ var sinceCommitIds = getProjectManager().readLfsSinceCommits(projectId);
+
+ if (sinceCommitIds.isEmpty()) {
+ new LfsFetchAllCommand(git.workingDir(), remoteUrl) {
+ protected Commandline newGit() {
+ return git;
+ }
+ }.run();
+ } else {
+ var fetchCommitIds = getReachableCommits(repository, sinceCommitIds, newCommitIds.values())
+ .stream().map(AnyObjectId::copy).collect(toList());
+ new LfsFetchCommand(git.workingDir(), remoteUrl, fetchCommitIds) {
+ @Override
+ protected Commandline newGit() {
+ return git;
+ }
+ }.run();
}
- for (Map.Entry<String, ValueDifference<ObjectId>> entry : difference.entriesDiffering().entrySet()) {
- if (RefUpdated.isValidRef(entry.getKey())) {
- registry.post(new RefUpdated(project, entry.getKey(),
- entry.getValue().leftValue(), entry.getValue().rightValue()));
+ getProjectManager().writeLfsSinceCommits(projectId, newCommitIds.values());
+ }
+
+ OneDev.getInstance(SessionManager.class).runAsync(() -> {
+ try {
+ // Access db connection in a separate thread to avoid possible deadlock, as
+ // the parent thread is blocking another thread holding database connections
+ var project = getProjectManager().load(projectId);
+ MapDifference<String, ObjectId> difference = difference(oldCommitIds, newCommitIds);
+ ListenerRegistry registry = OneDev.getInstance(ListenerRegistry.class);
+ for (Map.Entry<String, ObjectId> entry : difference.entriesOnlyOnLeft().entrySet()) {
+ if (RefUpdated.isValidRef(entry.getKey()))
+ registry.post(new RefUpdated(project, entry.getKey(), entry.getValue(), ObjectId.zeroId()));
}
+ for (Map.Entry<String, ObjectId> entry : difference.entriesOnlyOnRight().entrySet()) {
+ if (RefUpdated.isValidRef(entry.getKey()))
+ registry.post(new RefUpdated(project, entry.getKey(), ObjectId.zeroId(), entry.getValue()));
+ }
+ for (Map.Entry<String, ValueDifference<ObjectId>> entry : difference.entriesDiffering().entrySet()) {
+ if (RefUpdated.isValidRef(entry.getKey())) {
+ registry.post(new RefUpdated(project, entry.getKey(),
+ entry.getValue().leftValue(), entry.getValue().rightValue()));
+ }
+ }
+ } catch (Exception e) {
+ logger.error("Error posting ref updated event", e);
}
- } catch (Exception e) {
- logger.error("Error posting ref updated event", e);
- }
- });
+ });
- return null;
+ return null;
+ } finally {
+ if (certificateFile != null)
+ FileUtils.deleteFile(certificateFile);
+ }
}
}
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/PushRepository.java b/server-core/src/main/java/io/onedev/server/buildspec/step/PushRepository.java
index 74611d9ebc..8210c7fd00 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/step/PushRepository.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/PushRepository.java
@@ -1,5 +1,6 @@
package io.onedev.server.buildspec.step;
+import io.onedev.commons.utils.FileUtils;
import io.onedev.commons.utils.TaskLogger;
import io.onedev.commons.utils.command.Commandline;
import io.onedev.commons.utils.command.LineConsumer;
@@ -13,6 +14,7 @@ import io.onedev.server.model.Project;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Repository;
+import javax.annotation.Nullable;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.nio.charset.StandardCharsets;
@@ -27,46 +29,29 @@ public class PushRepository extends SyncRepository {
@Override
public Map<String, byte[]> run(Build build, File inputDir, TaskLogger logger) {
- if (OneDev.getInstance(ProjectManager.class).hasLfsObjects(build.getProject().getId())) {
- Project project = build.getProject();
- Commandline git = CommandUtils.newGit();
- git.workingDir(OneDev.getInstance(ProjectManager.class).getGitDir(project.getId()));
- configureProxy(git, getProxy());
-
- String remoteUrl = getRemoteUrlWithCredential(build);
- AtomicReference<String> remoteCommitId = new AtomicReference<>(null);
- git.addArgs("ls-remote", remoteUrl, "HEAD", build.getRefName());
- git.execute(new LineConsumer() {
-
- @Override
- public void consume(String line) {
- String refName = line.substring(40).trim();
- if (refName.equals("HEAD")) {
- if (remoteCommitId.get() == null)
- remoteCommitId.set(line.substring(0, 40));
- } else {
- remoteCommitId.set(line.substring(0, 40));
- }
- }
-
- }, new LineConsumer() {
-
- @Override
- public void consume(String line) {
- logger.warning(line);
- }
-
- });
-
- if (remoteCommitId.get() != null) {
- git.clearArgs();
+ var certificateFile = writeCertificate(getCertificate());
+ try {
+ if (OneDev.getInstance(ProjectManager.class).hasLfsObjects(build.getProject().getId())) {
+ Project project = build.getProject();
+ Commandline git = CommandUtils.newGit();
+ git.workingDir(OneDev.getInstance(ProjectManager.class).getGitDir(project.getId()));
configureProxy(git, getProxy());
- git.addArgs("fetch", remoteUrl, remoteCommitId.get());
+ configureCertificate(git, certificateFile);
+
+ String remoteUrl = getRemoteUrlWithCredential(build);
+ AtomicReference<String> remoteCommitId = new AtomicReference<>(null);
+ git.addArgs("ls-remote", remoteUrl, "HEAD", build.getRefName());
git.execute(new LineConsumer() {
@Override
public void consume(String line) {
- logger.log(line);
+ String refName = line.substring(40).trim();
+ if (refName.equals("HEAD")) {
+ if (remoteCommitId.get() == null)
+ remoteCommitId.set(line.substring(0, 40));
+ } else {
+ remoteCommitId.set(line.substring(0, 40));
+ }
}
}, new LineConsumer() {
@@ -76,86 +61,111 @@ public class PushRepository extends SyncRepository {
logger.warning(line);
}
- }).checkReturnCode();
-
- Repository repository = OneDev.getInstance(ProjectManager.class)
- .getRepository(project.getId());
- String mergeBaseId = GitUtils.getMergeBase(repository,
- ObjectId.fromString(remoteCommitId.get()), build.getCommitId()).name();
-
- if (!mergeBaseId.equals(build.getCommitHash())) {
- String input = String.format("%s %s %s %s\n", build.getRefName(), build.getCommitHash(),
- build.getRefName(), remoteCommitId.get());
+ });
+
+ if (remoteCommitId.get() != null) {
git.clearArgs();
configureProxy(git, getProxy());
- git.addArgs("lfs", "pre-push", remoteUrl, remoteUrl);
+ configureCertificate(git, certificateFile);
+ git.addArgs("fetch", remoteUrl, remoteCommitId.get());
git.execute(new LineConsumer() {
@Override
public void consume(String line) {
logger.log(line);
}
-
+
}, new LineConsumer() {
@Override
public void consume(String line) {
logger.warning(line);
}
-
- }, new ByteArrayInputStream(input.getBytes(StandardCharsets.UTF_8))).checkReturnCode();
- }
- } else {
- git.clearArgs();
- configureProxy(git, getProxy());
- git.addArgs("lfs", "push", "--all", remoteUrl, build.getCommitHash());
- git.execute(new LineConsumer() {
- @Override
- public void consume(String line) {
- logger.log(line);
- }
-
- }, new LineConsumer() {
+ }).checkReturnCode();
- @Override
- public void consume(String line) {
- logger.warning(line);
+ Repository repository = OneDev.getInstance(ProjectManager.class)
+ .getRepository(project.getId());
+ String mergeBaseId = GitUtils.getMergeBase(repository,
+ ObjectId.fromString(remoteCommitId.get()), build.getCommitId()).name();
+
+ if (!mergeBaseId.equals(build.getCommitHash())) {
+ String input = String.format("%s %s %s %s\n", build.getRefName(), build.getCommitHash(),
+ build.getRefName(), remoteCommitId.get());
+ git.clearArgs();
+ configureProxy(git, getProxy());
+ configureCertificate(git, certificateFile);
+ git.addArgs("lfs", "pre-push", remoteUrl, remoteUrl);
+ git.execute(new LineConsumer() {
+
+ @Override
+ public void consume(String line) {
+ logger.log(line);
+ }
+
+ }, new LineConsumer() {
+
+ @Override
+ public void consume(String line) {
+ logger.warning(line);
+ }
+
+ }, new ByteArrayInputStream(input.getBytes(StandardCharsets.UTF_8))).checkReturnCode();
}
-
- }).checkReturnCode();
- }
- }
- push(build, logger);
-
- return null;
- }
-
- private void push(Build build, TaskLogger logger) {
- Commandline git = CommandUtils.newGit();
- configureProxy(git, getProxy());
- git.workingDir(OneDev.getInstance(ProjectManager.class).getGitDir(build.getProject().getId()));
- git.addArgs("push");
- if (isForce())
- git.addArgs("--force");
- git.addArgs(getRemoteUrlWithCredential(build));
- git.addArgs(build.getCommitHash() + ":" + build.getRefName());
-
- git.execute(new LineConsumer() {
-
- @Override
- public void consume(String line) {
- logger.log(line);
- }
-
- }, new LineConsumer() {
+ } else {
+ git.clearArgs();
+ configureProxy(git, getProxy());
+ configureCertificate(git, certificateFile);
+ git.addArgs("lfs", "push", "--all", remoteUrl, build.getCommitHash());
+ git.execute(new LineConsumer() {
+
+ @Override
+ public void consume(String line) {
+ logger.log(line);
+ }
+
+ }, new LineConsumer() {
+
+ @Override
+ public void consume(String line) {
+ logger.warning(line);
+ }
- @Override
- public void consume(String line) {
- logger.warning(line);
+ }).checkReturnCode();
+ }
}
- }).checkReturnCode();
+ Commandline git = CommandUtils.newGit();
+ configureProxy(git, getProxy());
+ configureCertificate(git, certificateFile);
+ git.workingDir(OneDev.getInstance(ProjectManager.class).getGitDir(build.getProject().getId()));
+ git.addArgs("push");
+ if (isForce())
+ git.addArgs("--force");
+ git.addArgs(getRemoteUrlWithCredential(build));
+ git.addArgs(build.getCommitHash() + ":" + build.getRefName());
+
+ git.execute(new LineConsumer() {
+
+ @Override
+ public void consume(String line) {
+ logger.log(line);
+ }
+
+ }, new LineConsumer() {
+
+ @Override
+ public void consume(String line) {
+ logger.warning(line);
+ }
+
+ }).checkReturnCode();
+
+ return null;
+ } finally {
+ if (certificateFile != null)
+ FileUtils.deleteFile(certificateFile);
+ }
}
}
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/SyncRepository.java b/server-core/src/main/java/io/onedev/server/buildspec/step/SyncRepository.java
index b53a533872..53283e9f8e 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/step/SyncRepository.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/SyncRepository.java
@@ -2,12 +2,10 @@ package io.onedev.server.buildspec.step;
import io.onedev.commons.bootstrap.Bootstrap;
import io.onedev.commons.codeassist.InputSuggestion;
+import io.onedev.commons.utils.FileUtils;
import io.onedev.commons.utils.StringUtils;
import io.onedev.commons.utils.command.Commandline;
-import io.onedev.server.annotation.ChoiceProvider;
-import io.onedev.server.annotation.Editable;
-import io.onedev.server.annotation.Interpolative;
-import io.onedev.server.annotation.RegEx;
+import io.onedev.server.annotation.*;
import io.onedev.server.buildspec.BuildSpec;
import io.onedev.server.model.Build;
import io.onedev.server.model.Project;
@@ -15,6 +13,8 @@ import io.onedev.server.model.Project;
import javax.annotation.Nullable;
import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.Null;
+import java.io.File;
+import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
@@ -31,7 +31,9 @@ public abstract class SyncRepository extends ServerSideStep {
private String userName;
private String passwordSecret;
-
+
+ private String certificate;
+
private boolean force;
private String proxy;
@@ -79,6 +81,18 @@ public abstract class SyncRepository extends ServerSideStep {
return Project.get().getHierarchyJobSecrets()
.stream().map(it->it.getName()).collect(Collectors.toList());
}
+
+ @Editable(order=450, name="Certificate to Trust", placeholder = "Base64 encoded PEM format, starting with " +
+ "-----BEGIN CERTIFICATE----- and ending with -----END CERTIFICATE-----",
+ description = "Specify certificate to trust if you are using self-signed certificate for above url")
+ @Multiline
+ public String getCertificate() {
+ return certificate;
+ }
+
+ public void setCertificate(String certificate) {
+ this.certificate = certificate;
+ }
@SuppressWarnings("unused")
private static String getLfsDescription() {
@@ -143,4 +157,24 @@ public abstract class SyncRepository extends ServerSideStep {
git.addArgs("-c", "http.proxy=" + proxy, "-c", "https.proxy=" + proxy);
}
+ @Nullable
+ protected static File writeCertificate(@Nullable String certificate) {
+ if (certificate != null) {
+ try {
+ var file = File.createTempFile("certificate", "pem");
+ FileUtils.writeFile(file, certificate);
+ return file;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ return null;
+ }
+ }
+
+ protected static void configureCertificate(Commandline git, @Nullable File certificateFile) {
+ if (certificateFile != null)
+ git.addArgs("-c", "http.sslCAInfo=" + certificateFile.getAbsolutePath());
+ }
+
}