The Unified and Autonomous Development Platform
fix: pull from git mirror fails because self-signed cert (#1405)
Robin Shen089ce8e8856f
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/PullRepository.java b/server-core/src/main/java/io/onedev/server/buildspec/step/PullRepository.javaindex ccce9a2bcd..05d8cdd28c 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/step/PullRepository.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/PullRepository.java@@ -7,6 +7,7 @@ import edu.emory.mathcs.backport.java.util.Collections;import io.onedev.commons.bootstrap.Bootstrap;import io.onedev.commons.codeassist.InputSuggestion;import io.onedev.commons.utils.ExplicitException;+import io.onedev.commons.utils.FileUtils;import io.onedev.commons.utils.StringUtils;import io.onedev.commons.utils.TaskLogger;import io.onedev.commons.utils.command.Commandline;@@ -149,7 +150,7 @@ public class PullRepository extends SyncRepository {String remoteUrl = getRemoteUrlWithCredential(build);Long projectId = getTargetProject(build).getId();- var task = new PullTask(projectId, remoteUrl, getRefs(), isForce(), isWithLfs(), getProxy());+ var task = new PullTask(projectId, remoteUrl, getCertificate(), getRefs(), isForce(), isWithLfs(), getProxy());getProjectManager().runOnActiveServer(projectId, task);return null;@@ -178,6 +179,8 @@ public class PullRepository extends SyncRepository {private final String remoteUrl;+ private final String certificate;+private final String refs;private final boolean force;@@ -186,10 +189,11 @@ public class PullRepository extends SyncRepository {private final String proxy;- PullTask(Long projectId, String remoteUrl, String refs, boolean force,- boolean withLfs, @Nullable String proxy) {+ PullTask(Long projectId, String remoteUrl, @Nullable String certificate,+ String refs, boolean force, boolean withLfs, @Nullable String proxy) {this.projectId = projectId;this.remoteUrl = remoteUrl;+ this.certificate = certificate;this.refs = refs;this.force = force;this.withLfs = withLfs;@@ -214,88 +218,63 @@ public class PullRepository extends SyncRepository {@Overridepublic Void call() throws Exception {- Repository repository = getProjectManager().getRepository(projectId);-- String defaultBranch = GitUtils.getDefaultBranch(repository);- Map<String, ObjectId> oldCommitIds = getRefCommits(repository);-- Commandline git = CommandUtils.newGit();- configureProxy(git, proxy);- git.workingDir(repository.getDirectory());-- git.addArgs("fetch");- git.addArgs(remoteUrl);- if (force)- git.addArgs("--force");-- for (String each: Splitter.on(' ').omitEmptyStrings().trimResults().split(refs))- git.addArgs(each + ":" + each);-- git.execute(new LineConsumer() {-- @Override- public void consume(String line) {- logger.debug(line);- }-- }, new LineConsumer() {-- @Override- public void consume(String line) {- if (!line.startsWith("From") && !line.contains("->"))- logger.error(line);- else- logger.debug(line);- }+ var certificateFile = writeCertificate(certificate);+ try {+ Repository repository = getProjectManager().getRepository(projectId);- }).checkReturnCode();+ String defaultBranch = GitUtils.getDefaultBranch(repository);+ Map<String, ObjectId> oldCommitIds = getRefCommits(repository);- Map<String, ObjectId> newCommitIds = getRefCommits(repository);+ Commandline git = CommandUtils.newGit();+ configureProxy(git, proxy);+ configureCertificate(git, certificateFile);+ git.workingDir(repository.getDirectory());- if (defaultBranch == null) {- logger.debug("Determining remote head branch...");+ git.addArgs("fetch");+ git.addArgs(remoteUrl);+ if (force)+ git.addArgs("--force");- git.clearArgs();- configureProxy(git, proxy);- git.addArgs("remote", "show", remoteUrl);+ for (String each : Splitter.on(' ').omitEmptyStrings().trimResults().split(refs))+ git.addArgs(each + ":" + each);- AtomicReference<String> headBranch = new AtomicReference<>(null);git.execute(new LineConsumer() {@Overridepublic void consume(String line) {logger.debug(line);- if (line.trim().startsWith("HEAD branch:"))- headBranch.set(line.trim().substring("HEAD branch:".length()).trim());}}, new LineConsumer() {@Overridepublic void consume(String line) {- logger.warn(line);+ if (!line.startsWith("From") && !line.contains("->"))+ logger.error(line);+ else+ logger.debug(line);}}).checkReturnCode();- if (headBranch.get() != null) {- if (GitUtils.resolve(repository, Constants.R_HEADS + headBranch.get(), false) == null) {- logger.debug("Remote head branch not synced, using first branch as default");- headBranch.set(null);- }- } else {- logger.debug("Remote head branch not found, using first branch as default");- }- if (headBranch.get() == null) {+ Map<String, ObjectId> newCommitIds = getRefCommits(repository);++ if (defaultBranch == null) {+ logger.debug("Determining remote head branch...");+git.clearArgs();- git.addArgs("branch");+ configureProxy(git, proxy);+ configureCertificate(git, certificateFile);+ git.addArgs("remote", "show", remoteUrl);+ AtomicReference<String> headBranch = new AtomicReference<>(null);git.execute(new LineConsumer() {@Overridepublic void consume(String line) {- if (headBranch.get() == null)- headBranch.set(StringUtils.stripStart(line.trim(), "*").trim());+ logger.debug(line);+ if (line.trim().startsWith("HEAD branch:"))+ headBranch.set(line.trim().substring("HEAD branch:".length()).trim());}}, new LineConsumer() {@@ -306,62 +285,96 @@ public class PullRepository extends SyncRepository {}}).checkReturnCode();- }- if (headBranch.get() != null)- GitUtils.setDefaultBranch(repository, headBranch.get());- }-- if (withLfs) {- git.clearArgs();- configureProxy(git, proxy);- var sinceCommitIds = getProjectManager().readLfsSinceCommits(projectId);- if (sinceCommitIds.isEmpty()) {- new LfsFetchAllCommand(git.workingDir(), remoteUrl) {- protected Commandline newGit() {- return git;- }- }.run();- } else {- var fetchCommitIds = getReachableCommits(repository, sinceCommitIds, newCommitIds.values())- .stream().map(AnyObjectId::copy).collect(toList());- new LfsFetchCommand(git.workingDir(), remoteUrl, fetchCommitIds) {- @Override- protected Commandline newGit() {- return git;+ if (headBranch.get() != null) {+ if (GitUtils.resolve(repository, Constants.R_HEADS + headBranch.get(), false) == null) {+ logger.debug("Remote head branch not synced, using first branch as default");+ headBranch.set(null);}- }.run();- }- getProjectManager().writeLfsSinceCommits(projectId, newCommitIds.values());- }+ } else {+ logger.debug("Remote head branch not found, using first branch as default");+ }+ if (headBranch.get() == null) {+ git.clearArgs();+ git.addArgs("branch");++ git.execute(new LineConsumer() {++ @Override+ public void consume(String line) {+ if (headBranch.get() == null)+ headBranch.set(StringUtils.stripStart(line.trim(), "*").trim());+ }++ }, new LineConsumer() {++ @Override+ public void consume(String line) {+ logger.warn(line);+ }- OneDev.getInstance(SessionManager.class).runAsync(() -> {- try {- // Access db connection in a separate thread to avoid possible deadlock, as- // the parent thread is blocking another thread holding database connections- var project = getProjectManager().load(projectId);- MapDifference<String, ObjectId> difference = difference(oldCommitIds, newCommitIds);- ListenerRegistry registry = OneDev.getInstance(ListenerRegistry.class);- for (Map.Entry<String, ObjectId> entry : difference.entriesOnlyOnLeft().entrySet()) {- if (RefUpdated.isValidRef(entry.getKey()))- registry.post(new RefUpdated(project, entry.getKey(), entry.getValue(), ObjectId.zeroId()));+ }).checkReturnCode();}- for (Map.Entry<String, ObjectId> entry : difference.entriesOnlyOnRight().entrySet()) {- if (RefUpdated.isValidRef(entry.getKey()))- registry.post(new RefUpdated(project, entry.getKey(), ObjectId.zeroId(), entry.getValue()));+ if (headBranch.get() != null)+ GitUtils.setDefaultBranch(repository, headBranch.get());+ }++ if (withLfs) {+ git.clearArgs();+ configureProxy(git, proxy);+ configureCertificate(git, certificateFile);+ var sinceCommitIds = getProjectManager().readLfsSinceCommits(projectId);++ if (sinceCommitIds.isEmpty()) {+ new LfsFetchAllCommand(git.workingDir(), remoteUrl) {+ protected Commandline newGit() {+ return git;+ }+ }.run();+ } else {+ var fetchCommitIds = getReachableCommits(repository, sinceCommitIds, newCommitIds.values())+ .stream().map(AnyObjectId::copy).collect(toList());+ new LfsFetchCommand(git.workingDir(), remoteUrl, fetchCommitIds) {+ @Override+ protected Commandline newGit() {+ return git;+ }+ }.run();}- for (Map.Entry<String, ValueDifference<ObjectId>> entry : difference.entriesDiffering().entrySet()) {- if (RefUpdated.isValidRef(entry.getKey())) {- registry.post(new RefUpdated(project, entry.getKey(),- entry.getValue().leftValue(), entry.getValue().rightValue()));+ getProjectManager().writeLfsSinceCommits(projectId, newCommitIds.values());+ }++ OneDev.getInstance(SessionManager.class).runAsync(() -> {+ try {+ // Access db connection in a separate thread to avoid possible deadlock, as+ // the parent thread is blocking another thread holding database connections+ var project = getProjectManager().load(projectId);+ MapDifference<String, ObjectId> difference = difference(oldCommitIds, newCommitIds);+ ListenerRegistry registry = OneDev.getInstance(ListenerRegistry.class);+ for (Map.Entry<String, ObjectId> entry : difference.entriesOnlyOnLeft().entrySet()) {+ if (RefUpdated.isValidRef(entry.getKey()))+ registry.post(new RefUpdated(project, entry.getKey(), entry.getValue(), ObjectId.zeroId()));}+ for (Map.Entry<String, ObjectId> entry : difference.entriesOnlyOnRight().entrySet()) {+ if (RefUpdated.isValidRef(entry.getKey()))+ registry.post(new RefUpdated(project, entry.getKey(), ObjectId.zeroId(), entry.getValue()));+ }+ for (Map.Entry<String, ValueDifference<ObjectId>> entry : difference.entriesDiffering().entrySet()) {+ if (RefUpdated.isValidRef(entry.getKey())) {+ registry.post(new RefUpdated(project, entry.getKey(),+ entry.getValue().leftValue(), entry.getValue().rightValue()));+ }+ }+ } catch (Exception e) {+ logger.error("Error posting ref updated event", e);}- } catch (Exception e) {- logger.error("Error posting ref updated event", e);- }- });+ });- return null;+ return null;+ } finally {+ if (certificateFile != null)+ FileUtils.deleteFile(certificateFile);+ }}}diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/PushRepository.java b/server-core/src/main/java/io/onedev/server/buildspec/step/PushRepository.javaindex 74611d9ebc..8210c7fd00 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/step/PushRepository.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/PushRepository.java@@ -1,5 +1,6 @@package io.onedev.server.buildspec.step;+import io.onedev.commons.utils.FileUtils;import io.onedev.commons.utils.TaskLogger;import io.onedev.commons.utils.command.Commandline;import io.onedev.commons.utils.command.LineConsumer;@@ -13,6 +14,7 @@ import io.onedev.server.model.Project;import org.eclipse.jgit.lib.ObjectId;import org.eclipse.jgit.lib.Repository;+import javax.annotation.Nullable;import java.io.ByteArrayInputStream;import java.io.File;import java.nio.charset.StandardCharsets;@@ -27,46 +29,29 @@ public class PushRepository extends SyncRepository {@Overridepublic Map<String, byte[]> run(Build build, File inputDir, TaskLogger logger) {- if (OneDev.getInstance(ProjectManager.class).hasLfsObjects(build.getProject().getId())) {- Project project = build.getProject();- Commandline git = CommandUtils.newGit();- git.workingDir(OneDev.getInstance(ProjectManager.class).getGitDir(project.getId()));- configureProxy(git, getProxy());-- String remoteUrl = getRemoteUrlWithCredential(build);- AtomicReference<String> remoteCommitId = new AtomicReference<>(null);- git.addArgs("ls-remote", remoteUrl, "HEAD", build.getRefName());- git.execute(new LineConsumer() {-- @Override- public void consume(String line) {- String refName = line.substring(40).trim();- if (refName.equals("HEAD")) {- if (remoteCommitId.get() == null)- remoteCommitId.set(line.substring(0, 40));- } else {- remoteCommitId.set(line.substring(0, 40));- }- }-- }, new LineConsumer() {-- @Override- public void consume(String line) {- logger.warning(line);- }-- });-- if (remoteCommitId.get() != null) {- git.clearArgs();+ var certificateFile = writeCertificate(getCertificate());+ try {+ if (OneDev.getInstance(ProjectManager.class).hasLfsObjects(build.getProject().getId())) {+ Project project = build.getProject();+ Commandline git = CommandUtils.newGit();+ git.workingDir(OneDev.getInstance(ProjectManager.class).getGitDir(project.getId()));configureProxy(git, getProxy());- git.addArgs("fetch", remoteUrl, remoteCommitId.get());+ configureCertificate(git, certificateFile);++ String remoteUrl = getRemoteUrlWithCredential(build);+ AtomicReference<String> remoteCommitId = new AtomicReference<>(null);+ git.addArgs("ls-remote", remoteUrl, "HEAD", build.getRefName());git.execute(new LineConsumer() {@Overridepublic void consume(String line) {- logger.log(line);+ String refName = line.substring(40).trim();+ if (refName.equals("HEAD")) {+ if (remoteCommitId.get() == null)+ remoteCommitId.set(line.substring(0, 40));+ } else {+ remoteCommitId.set(line.substring(0, 40));+ }}}, new LineConsumer() {@@ -76,86 +61,111 @@ public class PushRepository extends SyncRepository {logger.warning(line);}- }).checkReturnCode();-- Repository repository = OneDev.getInstance(ProjectManager.class)- .getRepository(project.getId());- String mergeBaseId = GitUtils.getMergeBase(repository,- ObjectId.fromString(remoteCommitId.get()), build.getCommitId()).name();-- if (!mergeBaseId.equals(build.getCommitHash())) {- String input = String.format("%s %s %s %s\n", build.getRefName(), build.getCommitHash(),- build.getRefName(), remoteCommitId.get());+ });++ if (remoteCommitId.get() != null) {git.clearArgs();configureProxy(git, getProxy());- git.addArgs("lfs", "pre-push", remoteUrl, remoteUrl);+ configureCertificate(git, certificateFile);+ git.addArgs("fetch", remoteUrl, remoteCommitId.get());git.execute(new LineConsumer() {@Overridepublic void consume(String line) {logger.log(line);}-+}, new LineConsumer() {@Overridepublic void consume(String line) {logger.warning(line);}-- }, new ByteArrayInputStream(input.getBytes(StandardCharsets.UTF_8))).checkReturnCode();- }- } else {- git.clearArgs();- configureProxy(git, getProxy());- git.addArgs("lfs", "push", "--all", remoteUrl, build.getCommitHash());- git.execute(new LineConsumer() {- @Override- public void consume(String line) {- logger.log(line);- }-- }, new LineConsumer() {+ }).checkReturnCode();- @Override- public void consume(String line) {- logger.warning(line);+ Repository repository = OneDev.getInstance(ProjectManager.class)+ .getRepository(project.getId());+ String mergeBaseId = GitUtils.getMergeBase(repository,+ ObjectId.fromString(remoteCommitId.get()), build.getCommitId()).name();++ if (!mergeBaseId.equals(build.getCommitHash())) {+ String input = String.format("%s %s %s %s\n", build.getRefName(), build.getCommitHash(),+ build.getRefName(), remoteCommitId.get());+ git.clearArgs();+ configureProxy(git, getProxy());+ configureCertificate(git, certificateFile);+ git.addArgs("lfs", "pre-push", remoteUrl, remoteUrl);+ git.execute(new LineConsumer() {++ @Override+ public void consume(String line) {+ logger.log(line);+ }++ }, new LineConsumer() {++ @Override+ public void consume(String line) {+ logger.warning(line);+ }++ }, new ByteArrayInputStream(input.getBytes(StandardCharsets.UTF_8))).checkReturnCode();}-- }).checkReturnCode();- }- }- push(build, logger);-- return null;- }-- private void push(Build build, TaskLogger logger) {- Commandline git = CommandUtils.newGit();- configureProxy(git, getProxy());- git.workingDir(OneDev.getInstance(ProjectManager.class).getGitDir(build.getProject().getId()));- git.addArgs("push");- if (isForce())- git.addArgs("--force");- git.addArgs(getRemoteUrlWithCredential(build));- git.addArgs(build.getCommitHash() + ":" + build.getRefName());-- git.execute(new LineConsumer() {-- @Override- public void consume(String line) {- logger.log(line);- }-- }, new LineConsumer() {+ } else {+ git.clearArgs();+ configureProxy(git, getProxy());+ configureCertificate(git, certificateFile);+ git.addArgs("lfs", "push", "--all", remoteUrl, build.getCommitHash());+ git.execute(new LineConsumer() {++ @Override+ public void consume(String line) {+ logger.log(line);+ }++ }, new LineConsumer() {++ @Override+ public void consume(String line) {+ logger.warning(line);+ }- @Override- public void consume(String line) {- logger.warning(line);+ }).checkReturnCode();+ }}- }).checkReturnCode();+ Commandline git = CommandUtils.newGit();+ configureProxy(git, getProxy());+ configureCertificate(git, certificateFile);+ git.workingDir(OneDev.getInstance(ProjectManager.class).getGitDir(build.getProject().getId()));+ git.addArgs("push");+ if (isForce())+ git.addArgs("--force");+ git.addArgs(getRemoteUrlWithCredential(build));+ git.addArgs(build.getCommitHash() + ":" + build.getRefName());++ git.execute(new LineConsumer() {++ @Override+ public void consume(String line) {+ logger.log(line);+ }++ }, new LineConsumer() {++ @Override+ public void consume(String line) {+ logger.warning(line);+ }++ }).checkReturnCode();++ return null;+ } finally {+ if (certificateFile != null)+ FileUtils.deleteFile(certificateFile);+ }}}diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/SyncRepository.java b/server-core/src/main/java/io/onedev/server/buildspec/step/SyncRepository.javaindex b53a533872..53283e9f8e 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/step/SyncRepository.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/SyncRepository.java@@ -2,12 +2,10 @@ package io.onedev.server.buildspec.step;import io.onedev.commons.bootstrap.Bootstrap;import io.onedev.commons.codeassist.InputSuggestion;+import io.onedev.commons.utils.FileUtils;import io.onedev.commons.utils.StringUtils;import io.onedev.commons.utils.command.Commandline;-import io.onedev.server.annotation.ChoiceProvider;-import io.onedev.server.annotation.Editable;-import io.onedev.server.annotation.Interpolative;-import io.onedev.server.annotation.RegEx;+import io.onedev.server.annotation.*;import io.onedev.server.buildspec.BuildSpec;import io.onedev.server.model.Build;import io.onedev.server.model.Project;@@ -15,6 +13,8 @@ import io.onedev.server.model.Project;import javax.annotation.Nullable;import javax.validation.constraints.NotEmpty;import javax.validation.constraints.Null;+import java.io.File;+import java.io.IOException;import java.io.UnsupportedEncodingException;import java.net.URLEncoder;import java.nio.charset.StandardCharsets;@@ -31,7 +31,9 @@ public abstract class SyncRepository extends ServerSideStep {private String userName;private String passwordSecret;-++ private String certificate;+private boolean force;private String proxy;@@ -79,6 +81,18 @@ public abstract class SyncRepository extends ServerSideStep {return Project.get().getHierarchyJobSecrets().stream().map(it->it.getName()).collect(Collectors.toList());}++ @Editable(order=450, name="Certificate to Trust", placeholder = "Base64 encoded PEM format, starting with " ++ "-----BEGIN CERTIFICATE----- and ending with -----END CERTIFICATE-----",+ description = "Specify certificate to trust if you are using self-signed certificate for above url")+ @Multiline+ public String getCertificate() {+ return certificate;+ }++ public void setCertificate(String certificate) {+ this.certificate = certificate;+ }@SuppressWarnings("unused")private static String getLfsDescription() {@@ -143,4 +157,24 @@ public abstract class SyncRepository extends ServerSideStep {git.addArgs("-c", "http.proxy=" + proxy, "-c", "https.proxy=" + proxy);}+ @Nullable+ protected static File writeCertificate(@Nullable String certificate) {+ if (certificate != null) {+ try {+ var file = File.createTempFile("certificate", "pem");+ FileUtils.writeFile(file, certificate);+ return file;+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ } else {+ return null;+ }+ }++ protected static void configureCertificate(Commandline git, @Nullable File certificateFile) {+ if (certificateFile != null)+ git.addArgs("-c", "http.sslCAInfo=" + certificateFile.getAbsolutePath());+ }+}